node { name: "global_step/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@global_step" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 0 } } } } node { name: "global_step" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@global_step" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "shape" value { shape { } } } attr { key: "shared_name" value { s: "global_step" } } } node { name: "global_step/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "global_step" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_step/Assign" op: "AssignVariableOp" input: "global_step" input: "global_step/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "validate_shape" value { b: false } } } node { name: "global_step/Read/ReadVariableOp" op: "ReadVariableOp" input: "global_step" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } } node { name: "global_step/VarIsInitializedOp" op: "VarIsInitializedOp" input: "global_step" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_step/cond" op: "If" input: "global_step/VarIsInitializedOp" input: "global_step" input: "global_step/Initializer/zeros" attr { key: "Tcond" value { type: DT_BOOL } } attr { key: "Tin" value { list { type: DT_RESOURCE type: DT_INT64 } } } attr { key: "Tout" value { list { type: DT_INT64 } } } attr { key: "_lower_using_switch_merge" value { b: true } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "_read_only_resource_inputs" value { list { i: 1 } } } attr { key: "else_branch" value { func { name: "global_step_cond_false_7" } } } attr { key: "output_shapes" value { list { shape { } } } } attr { key: "then_branch" value { func { name: "global_step_cond_true_6" } } } } node { name: "global_step/cond/Identity" op: "Identity" input: "global_step/cond" attr { key: "T" value { type: DT_INT64 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_step/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 0 } } } } node { name: "global_step/add" op: "AddV2" input: "global_step/cond/Identity" input: "global_step/add/y" attr { key: "T" value { type: DT_INT64 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "Const" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "output/train.tf_record" } } } } node { name: "flat_filenames/shape" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: -1 } } } } node { name: "flat_filenames" op: "Reshape" input: "Const" input: "flat_filenames/shape" device: "/device:CPU:0" attr { key: "T" value { type: DT_STRING } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "TensorSliceDataset" op: "TensorSliceDataset" input: "flat_filenames" device: "/device:CPU:0" attr { key: "Toutput_types" value { list { type: DT_STRING } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "is_files" value { b: true } } attr { key: "metadata" value { s: "" } } attr { key: "output_shapes" value { list { shape { } } } } attr { key: "replicate_on_split" value { b: false } } experimental_type { type_id: TFT_PRODUCT args { type_id: TFT_DATASET args { type_id: TFT_PRODUCT args { type_id: TFT_TENSOR args { type_id: TFT_STRING } } } } } } node { name: "FlatMapDataset" op: "FlatMapDataset" input: "TensorSliceDataset" device: "/device:CPU:0" attr { key: "Targuments" value { list { } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "f" value { func { name: "__inference_Dataset_flat_map_read_one_file_30" attr { key: "_tf_data_function" value { b: true } } } } } attr { key: "metadata" value { s: "" } } attr { key: "output_shapes" value { list { shape { } } } } attr { key: "output_types" value { list { type: DT_STRING } } } experimental_type { type_id: TFT_PRODUCT args { type_id: TFT_DATASET args { type_id: TFT_PRODUCT args { type_id: TFT_TENSOR args { type_id: TFT_STRING } } } } } } node { name: "count" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: -1 } } } } node { name: "RepeatDataset" op: "RepeatDataset" input: "FlatMapDataset" input: "count" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "metadata" value { s: "" } } attr { key: "output_shapes" value { list { shape { } } } } attr { key: "output_types" value { list { type: DT_STRING } } } experimental_type { type_id: TFT_PRODUCT args { type_id: TFT_DATASET args { type_id: TFT_PRODUCT args { type_id: TFT_TENSOR args { type_id: TFT_STRING } } } } } } node { name: "buffer_size" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 100 } } } } node { name: "seed" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 0 } } } } node { name: "seed2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 0 } } } } node { name: "ShuffleDataset" op: "ShuffleDataset" input: "RepeatDataset" input: "buffer_size" input: "seed" input: "seed2" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "metadata" value { s: "" } } attr { key: "output_shapes" value { list { shape { } } } } attr { key: "output_types" value { list { type: DT_STRING } } } attr { key: "reshuffle_each_iteration" value { b: true } } experimental_type { type_id: TFT_PRODUCT args { type_id: TFT_DATASET args { type_id: TFT_PRODUCT args { type_id: TFT_TENSOR args { type_id: TFT_STRING } } } } } } node { name: "batch_size" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 16 } } } } node { name: "num_parallel_calls" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 16 } } } } node { name: "drop_remainder" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_BOOL } } attr { key: "value" value { tensor { dtype: DT_BOOL tensor_shape { } bool_val: true } } } } node { name: "MapAndBatchDataset" op: "MapAndBatchDataset" input: "ShuffleDataset" input: "batch_size" input: "num_parallel_calls" input: "drop_remainder" device: "/device:CPU:0" attr { key: "Targuments" value { list { } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "f" value { func { name: "__inference_tf_data_experimental_map_and_batch_lambda_64" attr { key: "_tf_data_function" value { b: true } } } } } attr { key: "metadata" value { s: "" } } attr { key: "output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } } shape { dim { size: 16 } dim { size: 300 } } shape { dim { size: 16 } } shape { dim { size: 16 } } shape { dim { size: 16 } dim { size: 300 } } } } } attr { key: "output_types" value { list { type: DT_INT32 type: DT_INT32 type: DT_INT32 type: DT_INT32 type: DT_INT32 } } } attr { key: "preserve_cardinality" value { b: true } } experimental_type { type_id: TFT_PRODUCT args { type_id: TFT_DATASET args { type_id: TFT_PRODUCT args { type_id: TFT_TENSOR args { type_id: TFT_INT32 } } args { type_id: TFT_TENSOR args { type_id: TFT_INT32 } } args { type_id: TFT_TENSOR args { type_id: TFT_INT32 } } args { type_id: TFT_TENSOR args { type_id: TFT_INT32 } } args { type_id: TFT_TENSOR args { type_id: TFT_INT32 } } } } } } node { name: "IteratorV2" op: "IteratorV2" device: "/device:CPU:0" attr { key: "_class" value { list { s: "loc:@MapAndBatchDataset" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "container" value { s: "" } } attr { key: "output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } } shape { dim { size: 16 } dim { size: 300 } } shape { dim { size: 16 } } shape { dim { size: 16 } } shape { dim { size: 16 } dim { size: 300 } } } } } attr { key: "output_types" value { list { type: DT_INT32 type: DT_INT32 type: DT_INT32 type: DT_INT32 type: DT_INT32 } } } attr { key: "shared_name" value { s: "" } } } node { name: "MakeIterator" op: "MakeIterator" input: "MapAndBatchDataset" input: "IteratorV2" device: "/device:CPU:0" attr { key: "_class" value { list { s: "loc:@MapAndBatchDataset" } } } experimental_type { type_id: TFT_PRODUCT } } node { name: "IteratorToStringHandle" op: "IteratorToStringHandle" input: "IteratorV2" device: "/device:CPU:0" attr { key: "_class" value { list { s: "loc:@MapAndBatchDataset" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "IteratorGetNext" op: "IteratorGetNext" input: "IteratorV2" device: "/device:CPU:0" attr { key: "_class" value { list { s: "loc:@MapAndBatchDataset" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } } shape { dim { size: 16 } dim { size: 300 } } shape { dim { size: 16 } } shape { dim { size: 16 } } shape { dim { size: 16 } dim { size: 300 } } } } } attr { key: "output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } } shape { dim { size: 16 } dim { size: 300 } } shape { dim { size: 16 } } shape { dim { size: 16 } } shape { dim { size: 16 } dim { size: 300 } } } } } attr { key: "output_types" value { list { type: DT_INT32 type: DT_INT32 type: DT_INT32 type: DT_INT32 type: DT_INT32 } } } } node { name: "Cast" op: "Cast" input: "IteratorGetNext:2" attr { key: "DstT" value { type: DT_FLOAT } } attr { key: "SrcT" value { type: DT_INT32 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } } } } } } node { name: "bert/embeddings/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: -1 } } } } node { name: "bert/embeddings/ExpandDims" op: "ExpandDims" input: "IteratorGetNext" input: "bert/embeddings/ExpandDims/dim" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 1 } } } } } } node { name: "bert/embeddings/word_embeddings/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\210R\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/word_embeddings/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/word_embeddings/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/embeddings/word_embeddings/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/embeddings/word_embeddings/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/embeddings/word_embeddings/Initializer/truncated_normal/mul" op: "Mul" input: "bert/embeddings/word_embeddings/Initializer/truncated_normal/TruncatedNormal" input: "bert/embeddings/word_embeddings/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/word_embeddings/Initializer/truncated_normal" op: "AddV2" input: "bert/embeddings/word_embeddings/Initializer/truncated_normal/mul" input: "bert/embeddings/word_embeddings/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "bert/embeddings/word_embeddings" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 21128 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/embeddings/word_embeddings" } } } node { name: "bert/embeddings/word_embeddings/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/embeddings/word_embeddings" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/word_embeddings/Assign" op: "AssignVariableOp" input: "bert/embeddings/word_embeddings" input: "bert/embeddings/word_embeddings/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/embeddings/word_embeddings/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/word_embeddings" attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: -1 } } } } node { name: "bert/embeddings/Reshape" op: "Reshape" input: "bert/embeddings/ExpandDims" input: "bert/embeddings/Reshape/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } } node { name: "bert/embeddings/Gather" op: "ResourceGather" input: "bert/embeddings/word_embeddings" input: "bert/embeddings/Reshape" attr { key: "Tindices" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "batch_dims" value { i: 0 } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_indices" value { b: true } } } node { name: "bert/embeddings/Identity" op: "Identity" input: "bert/embeddings/Gather" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/embeddings/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/Reshape_1" op: "Reshape" input: "bert/embeddings/Identity" input: "bert/embeddings/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\002\000\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/mul" op: "Mul" input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/TruncatedNormal" input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal" op: "AddV2" input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/mul" input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "bert/embeddings/token_type_embeddings" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 2 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/embeddings/token_type_embeddings" } } } node { name: "bert/embeddings/token_type_embeddings/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/embeddings/token_type_embeddings" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/token_type_embeddings/Assign" op: "AssignVariableOp" input: "bert/embeddings/token_type_embeddings" input: "bert/embeddings/token_type_embeddings/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/embeddings/token_type_embeddings/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/token_type_embeddings" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: -1 } } } } node { name: "bert/embeddings/Reshape_2" op: "Reshape" input: "IteratorGetNext:4" input: "bert/embeddings/Reshape_2/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } } node { name: "bert/embeddings/one_hot/on_value" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/embeddings/one_hot/off_value" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/one_hot/depth" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "bert/embeddings/one_hot" op: "OneHot" input: "bert/embeddings/Reshape_2" input: "bert/embeddings/one_hot/depth" input: "bert/embeddings/one_hot/on_value" input: "bert/embeddings/one_hot/off_value" attr { key: "T" value { type: DT_FLOAT } } attr { key: "TI" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 2 } } } } } attr { key: "axis" value { i: -1 } } } node { name: "bert/embeddings/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/token_type_embeddings" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/MatMul" op: "MatMul" input: "bert/embeddings/one_hot" input: "bert/embeddings/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/embeddings/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/Reshape_3" op: "Reshape" input: "bert/embeddings/MatMul" input: "bert/embeddings/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/embeddings/add" op: "AddV2" input: "bert/embeddings/Reshape_1" input: "bert/embeddings/Reshape_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/embeddings/assert_less_equal/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 300 } } } } node { name: "bert/embeddings/assert_less_equal/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 512 } } } } node { name: "bert/embeddings/assert_less_equal/LessEqual" op: "LessEqual" input: "bert/embeddings/assert_less_equal/x" input: "bert/embeddings/assert_less_equal/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/assert_less_equal/Rank" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "bert/embeddings/assert_less_equal/range/start" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "bert/embeddings/assert_less_equal/range/delta" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "bert/embeddings/assert_less_equal/range" op: "Range" input: "bert/embeddings/assert_less_equal/range/start" input: "bert/embeddings/assert_less_equal/Rank" input: "bert/embeddings/assert_less_equal/range/delta" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { } } } } } } node { name: "bert/embeddings/assert_less_equal/All" op: "All" input: "bert/embeddings/assert_less_equal/LessEqual" input: "bert/embeddings/assert_less_equal/range" attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "bert/embeddings/assert_less_equal/Assert/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "Condition x <= y did not hold element-wise:" } } } } node { name: "bert/embeddings/assert_less_equal/Assert/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "x (bert/embeddings/assert_less_equal/x:0) = " } } } } node { name: "bert/embeddings/assert_less_equal/Assert/Const_2" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "y (bert/embeddings/assert_less_equal/y:0) = " } } } } node { name: "bert/embeddings/assert_less_equal/Assert/Assert/data_0" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "Condition x <= y did not hold element-wise:" } } } } node { name: "bert/embeddings/assert_less_equal/Assert/Assert/data_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "x (bert/embeddings/assert_less_equal/x:0) = " } } } } node { name: "bert/embeddings/assert_less_equal/Assert/Assert/data_3" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "y (bert/embeddings/assert_less_equal/y:0) = " } } } } node { name: "bert/embeddings/assert_less_equal/Assert/Assert" op: "Assert" input: "bert/embeddings/assert_less_equal/All" input: "bert/embeddings/assert_less_equal/Assert/Assert/data_0" input: "bert/embeddings/assert_less_equal/Assert/Assert/data_1" input: "bert/embeddings/assert_less_equal/x" input: "bert/embeddings/assert_less_equal/Assert/Assert/data_3" input: "bert/embeddings/assert_less_equal/y" attr { key: "T" value { list { type: DT_STRING type: DT_STRING type: DT_INT32 type: DT_STRING type: DT_INT32 } } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "summarize" value { i: 3 } } } node { name: "bert/embeddings/position_embeddings/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\002\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/position_embeddings/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/position_embeddings/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/embeddings/position_embeddings/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/embeddings/position_embeddings/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/embeddings/position_embeddings/Initializer/truncated_normal/mul" op: "Mul" input: "bert/embeddings/position_embeddings/Initializer/truncated_normal/TruncatedNormal" input: "bert/embeddings/position_embeddings/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "bert/embeddings/position_embeddings/Initializer/truncated_normal" op: "AddV2" input: "bert/embeddings/position_embeddings/Initializer/truncated_normal/mul" input: "bert/embeddings/position_embeddings/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "bert/embeddings/position_embeddings" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 512 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/embeddings/position_embeddings" } } } node { name: "bert/embeddings/position_embeddings/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/embeddings/position_embeddings" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/position_embeddings/Assign" op: "AssignVariableOp" input: "bert/embeddings/position_embeddings" input: "bert/embeddings/position_embeddings/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/embeddings/position_embeddings/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/position_embeddings" attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/Slice/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/position_embeddings" input: "^bert/embeddings/assert_less_equal/Assert/Assert" attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/Slice/begin" op: "Const" input: "^bert/embeddings/assert_less_equal/Assert/Assert" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\000\000\000\000" } } } } node { name: "bert/embeddings/Slice/size" op: "Const" input: "^bert/embeddings/assert_less_equal/Assert/Assert" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: ",\001\000\000\377\377\377\377" } } } } node { name: "bert/embeddings/Slice" op: "Slice" input: "bert/embeddings/Slice/ReadVariableOp" input: "bert/embeddings/Slice/begin" input: "bert/embeddings/Slice/size" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/embeddings/Reshape_4/shape" op: "Const" input: "^bert/embeddings/assert_less_equal/Assert/Assert" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\001\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/Reshape_4" op: "Reshape" input: "bert/embeddings/Slice" input: "bert/embeddings/Reshape_4/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/embeddings/add_1" op: "AddV2" input: "bert/embeddings/add" input: "bert/embeddings/Reshape_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/embeddings/layer_normalization/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/layer_normalization/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/embeddings/layer_normalization/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/embeddings/layer_normalization/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/embeddings/layer_normalization/gamma" } } } node { name: "bert/embeddings/layer_normalization/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/layer_normalization/gamma/Assign" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/gamma" input: "bert/embeddings/layer_normalization/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/embeddings/layer_normalization/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/layer_normalization/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/layer_normalization/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/embeddings/layer_normalization/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/embeddings/layer_normalization/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/embeddings/layer_normalization/beta" } } } node { name: "bert/embeddings/layer_normalization/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/layer_normalization/beta/Assign" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/beta" input: "bert/embeddings/layer_normalization/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/embeddings/layer_normalization/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/layer_normalization/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 2 } } } } node { name: "bert/embeddings/layer_normalization/moments/mean" op: "Mean" input: "bert/embeddings/add_1" input: "bert/embeddings/layer_normalization/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/embeddings/layer_normalization/moments/StopGradient" op: "StopGradient" input: "bert/embeddings/layer_normalization/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 1 } } } } } } node { name: "bert/embeddings/layer_normalization/moments/SquaredDifference" op: "SquaredDifference" input: "bert/embeddings/add_1" input: "bert/embeddings/layer_normalization/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/embeddings/layer_normalization/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 2 } } } } node { name: "bert/embeddings/layer_normalization/moments/variance" op: "Mean" input: "bert/embeddings/layer_normalization/moments/SquaredDifference" input: "bert/embeddings/layer_normalization/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/embeddings/layer_normalization/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/embeddings/layer_normalization/batchnorm/add" op: "AddV2" input: "bert/embeddings/layer_normalization/moments/variance" input: "bert/embeddings/layer_normalization/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 1 } } } } } } node { name: "bert/embeddings/layer_normalization/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/embeddings/layer_normalization/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 1 } } } } } } node { name: "bert/embeddings/layer_normalization/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/layer_normalization/batchnorm/mul" op: "Mul" input: "bert/embeddings/layer_normalization/batchnorm/Rsqrt" input: "bert/embeddings/layer_normalization/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/embeddings/layer_normalization/batchnorm/mul_1" op: "Mul" input: "bert/embeddings/add_1" input: "bert/embeddings/layer_normalization/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/embeddings/layer_normalization/batchnorm/mul_2" op: "Mul" input: "bert/embeddings/layer_normalization/moments/mean" input: "bert/embeddings/layer_normalization/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/embeddings/layer_normalization/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/layer_normalization/batchnorm/sub" op: "Sub" input: "bert/embeddings/layer_normalization/batchnorm/ReadVariableOp" input: "bert/embeddings/layer_normalization/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/embeddings/layer_normalization/batchnorm/add_1" op: "AddV2" input: "bert/embeddings/layer_normalization/batchnorm/mul_1" input: "bert/embeddings/layer_normalization/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/embeddings/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/embeddings/dropout/Mul" op: "Mul" input: "bert/embeddings/layer_normalization/batchnorm/add_1" input: "bert/embeddings/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/embeddings/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/embeddings/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/embeddings/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/embeddings/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/embeddings/dropout/random_uniform/RandomUniform" input: "bert/embeddings/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/embeddings/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/dropout/SelectV2" op: "SelectV2" input: "bert/embeddings/dropout/GreaterEqual" input: "bert/embeddings/dropout/Mul" input: "bert/embeddings/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000\001\000\000\000,\001\000\000" } } } } node { name: "bert/encoder/Reshape" op: "Reshape" input: "IteratorGetNext:1" input: "bert/encoder/Reshape/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } } } } } } node { name: "bert/encoder/Cast" op: "Cast" input: "bert/encoder/Reshape" attr { key: "DstT" value { type: DT_FLOAT } } attr { key: "SrcT" value { type: DT_INT32 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } } } } } } node { name: "bert/encoder/ones/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\001\000\000\000" } } } } node { name: "bert/encoder/ones/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/ones" op: "Fill" input: "bert/encoder/ones/shape_as_tensor" input: "bert/encoder/ones/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 1 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/mul" op: "Mul" input: "bert/encoder/ones" input: "bert/encoder/Cast" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\377\377\377\377\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_1" op: "Reshape" input: "bert/embeddings/dropout/SelectV2" input: "bert/encoder/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/query/kernel" } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel" input: "bert/encoder/layer_0/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/query/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/query/bias" } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias" input: "bert/encoder/layer_0/attention/self/query/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/Reshape_1" input: "bert/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/query/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_0/attention/self/query/MatMul" input: "bert/encoder/layer_0/attention/self/query/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/key/kernel" } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel" input: "bert/encoder/layer_0/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/key/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/key/bias" } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias" input: "bert/encoder/layer_0/attention/self/key/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/Reshape_1" input: "bert/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/key/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_0/attention/self/key/MatMul" input: "bert/encoder/layer_0/attention/self/key/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/value/kernel" } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel" input: "bert/encoder/layer_0/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/value/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/value/bias" } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias" input: "bert/encoder/layer_0/attention/self/value/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/Reshape_1" input: "bert/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/value/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_0/attention/self/value/MatMul" input: "bert/encoder/layer_0/attention/self/value/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_0/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_0/attention/self/query/BiasAdd" input: "bert/encoder/layer_0/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_0/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_0/attention/self/Reshape" input: "bert/encoder/layer_0/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_0/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_0/attention/self/key/BiasAdd" input: "bert/encoder/layer_0/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_0/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_0/attention/self/Reshape_1" input: "bert/encoder/layer_0/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_0/attention/self/MatMul" op: "BatchMatMulV2" input: "bert/encoder/layer_0/attention/self/transpose" input: "bert/encoder/layer_0/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_0/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_0/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_0/attention/self/MatMul" input: "bert/encoder/layer_0/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_0/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_0/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_0/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_0/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_0/attention/self/sub" op: "Sub" input: "bert/encoder/layer_0/attention/self/sub/x" input: "bert/encoder/layer_0/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_0/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_0/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_0/attention/self/sub" input: "bert/encoder/layer_0/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_0/attention/self/add" op: "AddV2" input: "bert/encoder/layer_0/attention/self/Mul" input: "bert/encoder/layer_0/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_0/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_0/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/Mul" op: "Mul" input: "bert/encoder/layer_0/attention/self/Softmax" input: "bert/encoder/layer_0/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_0/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/attention/self/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_0/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_0/attention/self/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_0/attention/self/dropout/GreaterEqual" input: "bert/encoder/layer_0/attention/self/dropout/Mul" input: "bert/encoder/layer_0/attention/self/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_0/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_0/attention/self/value/BiasAdd" input: "bert/encoder/layer_0/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_0/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_0/attention/self/Reshape_2" input: "bert/encoder/layer_0/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_0/attention/self/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_0/attention/self/dropout/SelectV2" input: "bert/encoder/layer_0/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_0/attention/self/MatMul_1" input: "bert/encoder/layer_0/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_0/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_0/attention/self/transpose_3" input: "bert/encoder/layer_0/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/output/dense/kernel" } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel" input: "bert/encoder/layer_0/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/output/dense/bias" } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias" input: "bert/encoder/layer_0/attention/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_0/attention/self/Reshape_3" input: "bert/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_0/attention/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_0/attention/output/dense/MatMul" input: "bert/encoder/layer_0/attention/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_0/attention/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_0/attention/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_0/attention/output/dense/BiasAdd" input: "bert/encoder/layer_0/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_0/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/attention/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_0/attention/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_0/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_0/attention/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_0/attention/output/dropout/GreaterEqual" input: "bert/encoder/layer_0/attention/output/dropout/Mul" input: "bert/encoder/layer_0/attention/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/add" op: "AddV2" input: "bert/encoder/layer_0/attention/output/dropout/SelectV2" input: "bert/encoder/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/layer_normalization_1/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/layer_normalization_1/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta" } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean" op: "Mean" input: "bert/encoder/layer_0/attention/output/add" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_0/attention/output/add" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance" op: "Mean" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/Rsqrt" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_0/attention/output/add" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_1" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/intermediate/dense/kernel" } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel" input: "bert/encoder/layer_0/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/intermediate/dense/bias" } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias" input: "bert/encoder/layer_0/intermediate/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/add_1" input: "bert/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_0/intermediate/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_0/intermediate/dense/MatMul" input: "bert/encoder/layer_0/intermediate/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_0/intermediate/dense/Pow/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 3.0 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/Pow" op: "Pow" input: "bert/encoder/layer_0/intermediate/dense/BiasAdd" input: "bert/encoder/layer_0/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.044714998453855515 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_0/intermediate/dense/mul/x" input: "bert/encoder/layer_0/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/add" op: "AddV2" input: "bert/encoder/layer_0/intermediate/dense/BiasAdd" input: "bert/encoder/layer_0/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/mul_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.7978845834732056 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_0/intermediate/dense/mul_1/x" input: "bert/encoder/layer_0/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/Tanh" op: "Tanh" input: "bert/encoder/layer_0/intermediate/dense/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/add_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/add_1" op: "AddV2" input: "bert/encoder/layer_0/intermediate/dense/add_1/x" input: "bert/encoder/layer_0/intermediate/dense/Tanh" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/mul_2/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/mul_2" op: "Mul" input: "bert/encoder/layer_0/intermediate/dense/mul_2/x" input: "bert/encoder/layer_0/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/mul_3" op: "Mul" input: "bert/encoder/layer_0/intermediate/dense/BiasAdd" input: "bert/encoder/layer_0/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/output/dense/kernel" } } } node { name: "bert/encoder/layer_0/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/kernel" input: "bert/encoder/layer_0/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/output/dense/bias" } } } node { name: "bert/encoder/layer_0/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/bias" input: "bert/encoder/layer_0/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_0/intermediate/dense/mul_3" input: "bert/encoder/layer_0/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_0/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_0/output/dense/MatMul" input: "bert/encoder/layer_0/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_0/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_0/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_0/output/dense/BiasAdd" input: "bert/encoder/layer_0/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_0/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_0/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_0/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_0/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_0/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_0/output/dropout/GreaterEqual" input: "bert/encoder/layer_0/output/dropout/Mul" input: "bert/encoder/layer_0/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/add" op: "AddV2" input: "bert/encoder/layer_0/output/dropout/SelectV2" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/layer_normalization_2/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/layer_normalization_2/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/output/layer_normalization_2/gamma" } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/layer_normalization_2/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/layer_normalization_2/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/output/layer_normalization_2/beta" } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/moments/mean" op: "Mean" input: "bert/encoder/layer_0/output/add" input: "bert/encoder/layer_0/output/layer_normalization_2/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_0/output/layer_normalization_2/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_0/output/add" input: "bert/encoder/layer_0/output/layer_normalization_2/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/moments/variance" op: "Mean" input: "bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference" input: "bert/encoder/layer_0/output/layer_normalization_2/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_0/output/layer_normalization_2/moments/variance" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/Rsqrt" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_0/output/add" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_0/output/layer_normalization_2/moments/mean" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_1" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/query/kernel" } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel" input: "bert/encoder/layer_1/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/query/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/query/bias" } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias" input: "bert/encoder/layer_1/attention/self/query/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_1" input: "bert/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/query/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_1/attention/self/query/MatMul" input: "bert/encoder/layer_1/attention/self/query/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/key/kernel" } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel" input: "bert/encoder/layer_1/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/key/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/key/bias" } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias" input: "bert/encoder/layer_1/attention/self/key/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_1" input: "bert/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/key/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_1/attention/self/key/MatMul" input: "bert/encoder/layer_1/attention/self/key/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/value/kernel" } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel" input: "bert/encoder/layer_1/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/value/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/value/bias" } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias" input: "bert/encoder/layer_1/attention/self/value/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_1" input: "bert/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/value/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_1/attention/self/value/MatMul" input: "bert/encoder/layer_1/attention/self/value/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_1/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_1/attention/self/query/BiasAdd" input: "bert/encoder/layer_1/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_1/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_1/attention/self/Reshape" input: "bert/encoder/layer_1/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_1/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_1/attention/self/key/BiasAdd" input: "bert/encoder/layer_1/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_1/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_1/attention/self/Reshape_1" input: "bert/encoder/layer_1/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_1/attention/self/MatMul" op: "BatchMatMulV2" input: "bert/encoder/layer_1/attention/self/transpose" input: "bert/encoder/layer_1/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_1/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_1/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_1/attention/self/MatMul" input: "bert/encoder/layer_1/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_1/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_1/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_1/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_1/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_1/attention/self/sub" op: "Sub" input: "bert/encoder/layer_1/attention/self/sub/x" input: "bert/encoder/layer_1/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_1/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_1/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_1/attention/self/sub" input: "bert/encoder/layer_1/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_1/attention/self/add" op: "AddV2" input: "bert/encoder/layer_1/attention/self/Mul" input: "bert/encoder/layer_1/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_1/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_1/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/Mul" op: "Mul" input: "bert/encoder/layer_1/attention/self/Softmax" input: "bert/encoder/layer_1/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_1/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/attention/self/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_1/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_1/attention/self/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_1/attention/self/dropout/GreaterEqual" input: "bert/encoder/layer_1/attention/self/dropout/Mul" input: "bert/encoder/layer_1/attention/self/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_1/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_1/attention/self/value/BiasAdd" input: "bert/encoder/layer_1/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_1/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_1/attention/self/Reshape_2" input: "bert/encoder/layer_1/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_1/attention/self/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_1/attention/self/dropout/SelectV2" input: "bert/encoder/layer_1/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_1/attention/self/MatMul_1" input: "bert/encoder/layer_1/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_1/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_1/attention/self/transpose_3" input: "bert/encoder/layer_1/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/output/dense/kernel" } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel" input: "bert/encoder/layer_1/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/output/dense/bias" } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias" input: "bert/encoder/layer_1/attention/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_1/attention/self/Reshape_3" input: "bert/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_1/attention/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_1/attention/output/dense/MatMul" input: "bert/encoder/layer_1/attention/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_1/attention/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_1/attention/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_1/attention/output/dense/BiasAdd" input: "bert/encoder/layer_1/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_1/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/attention/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_1/attention/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_1/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_1/attention/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_1/attention/output/dropout/GreaterEqual" input: "bert/encoder/layer_1/attention/output/dropout/Mul" input: "bert/encoder/layer_1/attention/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/add" op: "AddV2" input: "bert/encoder/layer_1/attention/output/dropout/SelectV2" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/layer_normalization_3/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/layer_normalization_3/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta" } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean" op: "Mean" input: "bert/encoder/layer_1/attention/output/add" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_1/attention/output/add" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance" op: "Mean" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/Rsqrt" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_1/attention/output/add" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_1" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/intermediate/dense/kernel" } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel" input: "bert/encoder/layer_1/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/intermediate/dense/bias" } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias" input: "bert/encoder/layer_1/intermediate/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/add_1" input: "bert/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_1/intermediate/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_1/intermediate/dense/MatMul" input: "bert/encoder/layer_1/intermediate/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_1/intermediate/dense/Pow/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 3.0 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/Pow" op: "Pow" input: "bert/encoder/layer_1/intermediate/dense/BiasAdd" input: "bert/encoder/layer_1/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.044714998453855515 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_1/intermediate/dense/mul/x" input: "bert/encoder/layer_1/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/add" op: "AddV2" input: "bert/encoder/layer_1/intermediate/dense/BiasAdd" input: "bert/encoder/layer_1/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/mul_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.7978845834732056 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_1/intermediate/dense/mul_1/x" input: "bert/encoder/layer_1/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/Tanh" op: "Tanh" input: "bert/encoder/layer_1/intermediate/dense/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/add_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/add_1" op: "AddV2" input: "bert/encoder/layer_1/intermediate/dense/add_1/x" input: "bert/encoder/layer_1/intermediate/dense/Tanh" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/mul_2/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/mul_2" op: "Mul" input: "bert/encoder/layer_1/intermediate/dense/mul_2/x" input: "bert/encoder/layer_1/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/mul_3" op: "Mul" input: "bert/encoder/layer_1/intermediate/dense/BiasAdd" input: "bert/encoder/layer_1/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/output/dense/kernel" } } } node { name: "bert/encoder/layer_1/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/kernel" input: "bert/encoder/layer_1/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/output/dense/bias" } } } node { name: "bert/encoder/layer_1/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/bias" input: "bert/encoder/layer_1/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_1/intermediate/dense/mul_3" input: "bert/encoder/layer_1/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_1/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_1/output/dense/MatMul" input: "bert/encoder/layer_1/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_1/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_1/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_1/output/dense/BiasAdd" input: "bert/encoder/layer_1/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_1/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_1/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_1/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_1/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_1/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_1/output/dropout/GreaterEqual" input: "bert/encoder/layer_1/output/dropout/Mul" input: "bert/encoder/layer_1/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/add" op: "AddV2" input: "bert/encoder/layer_1/output/dropout/SelectV2" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/layer_normalization_4/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/layer_normalization_4/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/output/layer_normalization_4/gamma" } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/layer_normalization_4/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/layer_normalization_4/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/output/layer_normalization_4/beta" } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/moments/mean" op: "Mean" input: "bert/encoder/layer_1/output/add" input: "bert/encoder/layer_1/output/layer_normalization_4/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_1/output/layer_normalization_4/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_1/output/add" input: "bert/encoder/layer_1/output/layer_normalization_4/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/moments/variance" op: "Mean" input: "bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference" input: "bert/encoder/layer_1/output/layer_normalization_4/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_1/output/layer_normalization_4/moments/variance" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/Rsqrt" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_1/output/add" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_1/output/layer_normalization_4/moments/mean" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_1" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/query/kernel" } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel" input: "bert/encoder/layer_2/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/query/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/query/bias" } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias" input: "bert/encoder/layer_2/attention/self/query/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_1" input: "bert/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/query/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_2/attention/self/query/MatMul" input: "bert/encoder/layer_2/attention/self/query/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/key/kernel" } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel" input: "bert/encoder/layer_2/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/key/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/key/bias" } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias" input: "bert/encoder/layer_2/attention/self/key/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_1" input: "bert/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/key/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_2/attention/self/key/MatMul" input: "bert/encoder/layer_2/attention/self/key/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/value/kernel" } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel" input: "bert/encoder/layer_2/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/value/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/value/bias" } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias" input: "bert/encoder/layer_2/attention/self/value/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_1" input: "bert/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/value/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_2/attention/self/value/MatMul" input: "bert/encoder/layer_2/attention/self/value/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_2/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_2/attention/self/query/BiasAdd" input: "bert/encoder/layer_2/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_2/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_2/attention/self/Reshape" input: "bert/encoder/layer_2/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_2/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_2/attention/self/key/BiasAdd" input: "bert/encoder/layer_2/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_2/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_2/attention/self/Reshape_1" input: "bert/encoder/layer_2/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_2/attention/self/MatMul" op: "BatchMatMulV2" input: "bert/encoder/layer_2/attention/self/transpose" input: "bert/encoder/layer_2/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_2/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_2/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_2/attention/self/MatMul" input: "bert/encoder/layer_2/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_2/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_2/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_2/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_2/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_2/attention/self/sub" op: "Sub" input: "bert/encoder/layer_2/attention/self/sub/x" input: "bert/encoder/layer_2/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_2/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_2/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_2/attention/self/sub" input: "bert/encoder/layer_2/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_2/attention/self/add" op: "AddV2" input: "bert/encoder/layer_2/attention/self/Mul" input: "bert/encoder/layer_2/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_2/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_2/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/Mul" op: "Mul" input: "bert/encoder/layer_2/attention/self/Softmax" input: "bert/encoder/layer_2/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_2/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/attention/self/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_2/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_2/attention/self/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_2/attention/self/dropout/GreaterEqual" input: "bert/encoder/layer_2/attention/self/dropout/Mul" input: "bert/encoder/layer_2/attention/self/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_2/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_2/attention/self/value/BiasAdd" input: "bert/encoder/layer_2/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_2/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_2/attention/self/Reshape_2" input: "bert/encoder/layer_2/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_2/attention/self/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_2/attention/self/dropout/SelectV2" input: "bert/encoder/layer_2/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_2/attention/self/MatMul_1" input: "bert/encoder/layer_2/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_2/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_2/attention/self/transpose_3" input: "bert/encoder/layer_2/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/output/dense/kernel" } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel" input: "bert/encoder/layer_2/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/output/dense/bias" } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias" input: "bert/encoder/layer_2/attention/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_2/attention/self/Reshape_3" input: "bert/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_2/attention/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_2/attention/output/dense/MatMul" input: "bert/encoder/layer_2/attention/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_2/attention/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_2/attention/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_2/attention/output/dense/BiasAdd" input: "bert/encoder/layer_2/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_2/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/attention/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_2/attention/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_2/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_2/attention/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_2/attention/output/dropout/GreaterEqual" input: "bert/encoder/layer_2/attention/output/dropout/Mul" input: "bert/encoder/layer_2/attention/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/add" op: "AddV2" input: "bert/encoder/layer_2/attention/output/dropout/SelectV2" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/layer_normalization_5/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/layer_normalization_5/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta" } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean" op: "Mean" input: "bert/encoder/layer_2/attention/output/add" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_2/attention/output/add" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance" op: "Mean" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/Rsqrt" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_2/attention/output/add" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_1" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/intermediate/dense/kernel" } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel" input: "bert/encoder/layer_2/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/intermediate/dense/bias" } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias" input: "bert/encoder/layer_2/intermediate/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/add_1" input: "bert/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_2/intermediate/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_2/intermediate/dense/MatMul" input: "bert/encoder/layer_2/intermediate/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_2/intermediate/dense/Pow/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 3.0 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/Pow" op: "Pow" input: "bert/encoder/layer_2/intermediate/dense/BiasAdd" input: "bert/encoder/layer_2/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.044714998453855515 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_2/intermediate/dense/mul/x" input: "bert/encoder/layer_2/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/add" op: "AddV2" input: "bert/encoder/layer_2/intermediate/dense/BiasAdd" input: "bert/encoder/layer_2/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/mul_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.7978845834732056 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_2/intermediate/dense/mul_1/x" input: "bert/encoder/layer_2/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/Tanh" op: "Tanh" input: "bert/encoder/layer_2/intermediate/dense/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/add_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/add_1" op: "AddV2" input: "bert/encoder/layer_2/intermediate/dense/add_1/x" input: "bert/encoder/layer_2/intermediate/dense/Tanh" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/mul_2/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/mul_2" op: "Mul" input: "bert/encoder/layer_2/intermediate/dense/mul_2/x" input: "bert/encoder/layer_2/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/mul_3" op: "Mul" input: "bert/encoder/layer_2/intermediate/dense/BiasAdd" input: "bert/encoder/layer_2/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/output/dense/kernel" } } } node { name: "bert/encoder/layer_2/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/kernel" input: "bert/encoder/layer_2/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/output/dense/bias" } } } node { name: "bert/encoder/layer_2/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/bias" input: "bert/encoder/layer_2/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_2/intermediate/dense/mul_3" input: "bert/encoder/layer_2/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_2/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_2/output/dense/MatMul" input: "bert/encoder/layer_2/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_2/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_2/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_2/output/dense/BiasAdd" input: "bert/encoder/layer_2/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_2/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_2/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_2/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_2/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_2/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_2/output/dropout/GreaterEqual" input: "bert/encoder/layer_2/output/dropout/Mul" input: "bert/encoder/layer_2/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/add" op: "AddV2" input: "bert/encoder/layer_2/output/dropout/SelectV2" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/layer_normalization_6/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/layer_normalization_6/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/output/layer_normalization_6/gamma" } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/layer_normalization_6/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/layer_normalization_6/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/output/layer_normalization_6/beta" } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/moments/mean" op: "Mean" input: "bert/encoder/layer_2/output/add" input: "bert/encoder/layer_2/output/layer_normalization_6/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_2/output/layer_normalization_6/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_2/output/add" input: "bert/encoder/layer_2/output/layer_normalization_6/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/moments/variance" op: "Mean" input: "bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference" input: "bert/encoder/layer_2/output/layer_normalization_6/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_2/output/layer_normalization_6/moments/variance" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/Rsqrt" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_2/output/add" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_2/output/layer_normalization_6/moments/mean" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_1" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/query/kernel" } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel" input: "bert/encoder/layer_3/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/query/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/query/bias" } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias" input: "bert/encoder/layer_3/attention/self/query/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_1" input: "bert/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/query/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_3/attention/self/query/MatMul" input: "bert/encoder/layer_3/attention/self/query/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/key/kernel" } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel" input: "bert/encoder/layer_3/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/key/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/key/bias" } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias" input: "bert/encoder/layer_3/attention/self/key/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_1" input: "bert/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/key/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_3/attention/self/key/MatMul" input: "bert/encoder/layer_3/attention/self/key/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/value/kernel" } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel" input: "bert/encoder/layer_3/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/value/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/value/bias" } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias" input: "bert/encoder/layer_3/attention/self/value/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_1" input: "bert/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/value/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_3/attention/self/value/MatMul" input: "bert/encoder/layer_3/attention/self/value/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_3/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_3/attention/self/query/BiasAdd" input: "bert/encoder/layer_3/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_3/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_3/attention/self/Reshape" input: "bert/encoder/layer_3/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_3/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_3/attention/self/key/BiasAdd" input: "bert/encoder/layer_3/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_3/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_3/attention/self/Reshape_1" input: "bert/encoder/layer_3/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_3/attention/self/MatMul" op: "BatchMatMulV2" input: "bert/encoder/layer_3/attention/self/transpose" input: "bert/encoder/layer_3/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_3/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_3/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_3/attention/self/MatMul" input: "bert/encoder/layer_3/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_3/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_3/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_3/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_3/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_3/attention/self/sub" op: "Sub" input: "bert/encoder/layer_3/attention/self/sub/x" input: "bert/encoder/layer_3/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_3/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_3/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_3/attention/self/sub" input: "bert/encoder/layer_3/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_3/attention/self/add" op: "AddV2" input: "bert/encoder/layer_3/attention/self/Mul" input: "bert/encoder/layer_3/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_3/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_3/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/Mul" op: "Mul" input: "bert/encoder/layer_3/attention/self/Softmax" input: "bert/encoder/layer_3/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_3/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/attention/self/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_3/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_3/attention/self/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_3/attention/self/dropout/GreaterEqual" input: "bert/encoder/layer_3/attention/self/dropout/Mul" input: "bert/encoder/layer_3/attention/self/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_3/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_3/attention/self/value/BiasAdd" input: "bert/encoder/layer_3/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_3/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_3/attention/self/Reshape_2" input: "bert/encoder/layer_3/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_3/attention/self/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_3/attention/self/dropout/SelectV2" input: "bert/encoder/layer_3/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_3/attention/self/MatMul_1" input: "bert/encoder/layer_3/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_3/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_3/attention/self/transpose_3" input: "bert/encoder/layer_3/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/output/dense/kernel" } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel" input: "bert/encoder/layer_3/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/output/dense/bias" } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias" input: "bert/encoder/layer_3/attention/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_3/attention/self/Reshape_3" input: "bert/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_3/attention/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_3/attention/output/dense/MatMul" input: "bert/encoder/layer_3/attention/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_3/attention/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_3/attention/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_3/attention/output/dense/BiasAdd" input: "bert/encoder/layer_3/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_3/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/attention/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_3/attention/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_3/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_3/attention/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_3/attention/output/dropout/GreaterEqual" input: "bert/encoder/layer_3/attention/output/dropout/Mul" input: "bert/encoder/layer_3/attention/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/add" op: "AddV2" input: "bert/encoder/layer_3/attention/output/dropout/SelectV2" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/layer_normalization_7/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/layer_normalization_7/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta" } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean" op: "Mean" input: "bert/encoder/layer_3/attention/output/add" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_3/attention/output/add" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance" op: "Mean" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/Rsqrt" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_3/attention/output/add" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_1" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/intermediate/dense/kernel" } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel" input: "bert/encoder/layer_3/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/intermediate/dense/bias" } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias" input: "bert/encoder/layer_3/intermediate/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/add_1" input: "bert/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_3/intermediate/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_3/intermediate/dense/MatMul" input: "bert/encoder/layer_3/intermediate/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_3/intermediate/dense/Pow/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 3.0 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/Pow" op: "Pow" input: "bert/encoder/layer_3/intermediate/dense/BiasAdd" input: "bert/encoder/layer_3/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.044714998453855515 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_3/intermediate/dense/mul/x" input: "bert/encoder/layer_3/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/add" op: "AddV2" input: "bert/encoder/layer_3/intermediate/dense/BiasAdd" input: "bert/encoder/layer_3/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/mul_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.7978845834732056 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_3/intermediate/dense/mul_1/x" input: "bert/encoder/layer_3/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/Tanh" op: "Tanh" input: "bert/encoder/layer_3/intermediate/dense/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/add_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/add_1" op: "AddV2" input: "bert/encoder/layer_3/intermediate/dense/add_1/x" input: "bert/encoder/layer_3/intermediate/dense/Tanh" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/mul_2/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/mul_2" op: "Mul" input: "bert/encoder/layer_3/intermediate/dense/mul_2/x" input: "bert/encoder/layer_3/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/mul_3" op: "Mul" input: "bert/encoder/layer_3/intermediate/dense/BiasAdd" input: "bert/encoder/layer_3/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/output/dense/kernel" } } } node { name: "bert/encoder/layer_3/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/kernel" input: "bert/encoder/layer_3/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/output/dense/bias" } } } node { name: "bert/encoder/layer_3/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/bias" input: "bert/encoder/layer_3/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_3/intermediate/dense/mul_3" input: "bert/encoder/layer_3/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_3/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_3/output/dense/MatMul" input: "bert/encoder/layer_3/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_3/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_3/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_3/output/dense/BiasAdd" input: "bert/encoder/layer_3/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_3/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_3/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_3/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_3/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_3/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_3/output/dropout/GreaterEqual" input: "bert/encoder/layer_3/output/dropout/Mul" input: "bert/encoder/layer_3/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/add" op: "AddV2" input: "bert/encoder/layer_3/output/dropout/SelectV2" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/layer_normalization_8/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/layer_normalization_8/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/output/layer_normalization_8/gamma" } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/layer_normalization_8/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/layer_normalization_8/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/output/layer_normalization_8/beta" } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/moments/mean" op: "Mean" input: "bert/encoder/layer_3/output/add" input: "bert/encoder/layer_3/output/layer_normalization_8/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_3/output/layer_normalization_8/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_3/output/add" input: "bert/encoder/layer_3/output/layer_normalization_8/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/moments/variance" op: "Mean" input: "bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference" input: "bert/encoder/layer_3/output/layer_normalization_8/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_3/output/layer_normalization_8/moments/variance" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/Rsqrt" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_3/output/add" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_3/output/layer_normalization_8/moments/mean" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_1" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/query/kernel" } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel" input: "bert/encoder/layer_4/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/query/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/query/bias" } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias" input: "bert/encoder/layer_4/attention/self/query/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_1" input: "bert/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/query/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_4/attention/self/query/MatMul" input: "bert/encoder/layer_4/attention/self/query/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/key/kernel" } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel" input: "bert/encoder/layer_4/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/key/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/key/bias" } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias" input: "bert/encoder/layer_4/attention/self/key/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_1" input: "bert/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/key/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_4/attention/self/key/MatMul" input: "bert/encoder/layer_4/attention/self/key/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/value/kernel" } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel" input: "bert/encoder/layer_4/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/value/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/value/bias" } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias" input: "bert/encoder/layer_4/attention/self/value/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_1" input: "bert/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/value/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_4/attention/self/value/MatMul" input: "bert/encoder/layer_4/attention/self/value/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_4/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_4/attention/self/query/BiasAdd" input: "bert/encoder/layer_4/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_4/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_4/attention/self/Reshape" input: "bert/encoder/layer_4/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_4/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_4/attention/self/key/BiasAdd" input: "bert/encoder/layer_4/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_4/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_4/attention/self/Reshape_1" input: "bert/encoder/layer_4/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_4/attention/self/MatMul" op: "BatchMatMulV2" input: "bert/encoder/layer_4/attention/self/transpose" input: "bert/encoder/layer_4/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_4/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_4/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_4/attention/self/MatMul" input: "bert/encoder/layer_4/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_4/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_4/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_4/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_4/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_4/attention/self/sub" op: "Sub" input: "bert/encoder/layer_4/attention/self/sub/x" input: "bert/encoder/layer_4/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_4/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_4/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_4/attention/self/sub" input: "bert/encoder/layer_4/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_4/attention/self/add" op: "AddV2" input: "bert/encoder/layer_4/attention/self/Mul" input: "bert/encoder/layer_4/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_4/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_4/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/Mul" op: "Mul" input: "bert/encoder/layer_4/attention/self/Softmax" input: "bert/encoder/layer_4/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_4/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/attention/self/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_4/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_4/attention/self/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_4/attention/self/dropout/GreaterEqual" input: "bert/encoder/layer_4/attention/self/dropout/Mul" input: "bert/encoder/layer_4/attention/self/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_4/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_4/attention/self/value/BiasAdd" input: "bert/encoder/layer_4/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_4/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_4/attention/self/Reshape_2" input: "bert/encoder/layer_4/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_4/attention/self/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_4/attention/self/dropout/SelectV2" input: "bert/encoder/layer_4/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_4/attention/self/MatMul_1" input: "bert/encoder/layer_4/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_4/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_4/attention/self/transpose_3" input: "bert/encoder/layer_4/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/output/dense/kernel" } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel" input: "bert/encoder/layer_4/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/output/dense/bias" } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias" input: "bert/encoder/layer_4/attention/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_4/attention/self/Reshape_3" input: "bert/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_4/attention/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_4/attention/output/dense/MatMul" input: "bert/encoder/layer_4/attention/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_4/attention/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_4/attention/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_4/attention/output/dense/BiasAdd" input: "bert/encoder/layer_4/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_4/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/attention/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_4/attention/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_4/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_4/attention/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_4/attention/output/dropout/GreaterEqual" input: "bert/encoder/layer_4/attention/output/dropout/Mul" input: "bert/encoder/layer_4/attention/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/add" op: "AddV2" input: "bert/encoder/layer_4/attention/output/dropout/SelectV2" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/layer_normalization_9/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/layer_normalization_9/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta" } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean" op: "Mean" input: "bert/encoder/layer_4/attention/output/add" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_4/attention/output/add" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance" op: "Mean" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/Rsqrt" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_4/attention/output/add" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_1" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/intermediate/dense/kernel" } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel" input: "bert/encoder/layer_4/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/intermediate/dense/bias" } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias" input: "bert/encoder/layer_4/intermediate/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/add_1" input: "bert/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_4/intermediate/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_4/intermediate/dense/MatMul" input: "bert/encoder/layer_4/intermediate/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_4/intermediate/dense/Pow/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 3.0 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/Pow" op: "Pow" input: "bert/encoder/layer_4/intermediate/dense/BiasAdd" input: "bert/encoder/layer_4/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.044714998453855515 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_4/intermediate/dense/mul/x" input: "bert/encoder/layer_4/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/add" op: "AddV2" input: "bert/encoder/layer_4/intermediate/dense/BiasAdd" input: "bert/encoder/layer_4/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/mul_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.7978845834732056 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_4/intermediate/dense/mul_1/x" input: "bert/encoder/layer_4/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/Tanh" op: "Tanh" input: "bert/encoder/layer_4/intermediate/dense/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/add_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/add_1" op: "AddV2" input: "bert/encoder/layer_4/intermediate/dense/add_1/x" input: "bert/encoder/layer_4/intermediate/dense/Tanh" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/mul_2/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/mul_2" op: "Mul" input: "bert/encoder/layer_4/intermediate/dense/mul_2/x" input: "bert/encoder/layer_4/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/mul_3" op: "Mul" input: "bert/encoder/layer_4/intermediate/dense/BiasAdd" input: "bert/encoder/layer_4/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/output/dense/kernel" } } } node { name: "bert/encoder/layer_4/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/kernel" input: "bert/encoder/layer_4/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/output/dense/bias" } } } node { name: "bert/encoder/layer_4/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/bias" input: "bert/encoder/layer_4/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_4/intermediate/dense/mul_3" input: "bert/encoder/layer_4/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_4/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_4/output/dense/MatMul" input: "bert/encoder/layer_4/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_4/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_4/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_4/output/dense/BiasAdd" input: "bert/encoder/layer_4/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_4/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_4/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_4/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_4/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_4/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_4/output/dropout/GreaterEqual" input: "bert/encoder/layer_4/output/dropout/Mul" input: "bert/encoder/layer_4/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/add" op: "AddV2" input: "bert/encoder/layer_4/output/dropout/SelectV2" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/layer_normalization_10/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/layer_normalization_10/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/output/layer_normalization_10/gamma" } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/layer_normalization_10/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/layer_normalization_10/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/output/layer_normalization_10/beta" } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/moments/mean" op: "Mean" input: "bert/encoder/layer_4/output/add" input: "bert/encoder/layer_4/output/layer_normalization_10/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_4/output/layer_normalization_10/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_4/output/add" input: "bert/encoder/layer_4/output/layer_normalization_10/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/moments/variance" op: "Mean" input: "bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference" input: "bert/encoder/layer_4/output/layer_normalization_10/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_4/output/layer_normalization_10/moments/variance" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/Rsqrt" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_4/output/add" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_4/output/layer_normalization_10/moments/mean" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_1" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/query/kernel" } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel" input: "bert/encoder/layer_5/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/query/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/query/bias" } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias" input: "bert/encoder/layer_5/attention/self/query/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_1" input: "bert/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/query/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_5/attention/self/query/MatMul" input: "bert/encoder/layer_5/attention/self/query/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/key/kernel" } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel" input: "bert/encoder/layer_5/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/key/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/key/bias" } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias" input: "bert/encoder/layer_5/attention/self/key/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_1" input: "bert/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/key/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_5/attention/self/key/MatMul" input: "bert/encoder/layer_5/attention/self/key/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/value/kernel" } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel" input: "bert/encoder/layer_5/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/value/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/value/bias" } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias" input: "bert/encoder/layer_5/attention/self/value/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_1" input: "bert/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/value/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_5/attention/self/value/MatMul" input: "bert/encoder/layer_5/attention/self/value/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_5/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_5/attention/self/query/BiasAdd" input: "bert/encoder/layer_5/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_5/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_5/attention/self/Reshape" input: "bert/encoder/layer_5/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_5/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_5/attention/self/key/BiasAdd" input: "bert/encoder/layer_5/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_5/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_5/attention/self/Reshape_1" input: "bert/encoder/layer_5/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_5/attention/self/MatMul" op: "BatchMatMulV2" input: "bert/encoder/layer_5/attention/self/transpose" input: "bert/encoder/layer_5/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_5/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_5/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_5/attention/self/MatMul" input: "bert/encoder/layer_5/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_5/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_5/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_5/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_5/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_5/attention/self/sub" op: "Sub" input: "bert/encoder/layer_5/attention/self/sub/x" input: "bert/encoder/layer_5/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_5/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_5/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_5/attention/self/sub" input: "bert/encoder/layer_5/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_5/attention/self/add" op: "AddV2" input: "bert/encoder/layer_5/attention/self/Mul" input: "bert/encoder/layer_5/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_5/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_5/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/Mul" op: "Mul" input: "bert/encoder/layer_5/attention/self/Softmax" input: "bert/encoder/layer_5/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_5/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/attention/self/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_5/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_5/attention/self/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_5/attention/self/dropout/GreaterEqual" input: "bert/encoder/layer_5/attention/self/dropout/Mul" input: "bert/encoder/layer_5/attention/self/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_5/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_5/attention/self/value/BiasAdd" input: "bert/encoder/layer_5/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_5/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_5/attention/self/Reshape_2" input: "bert/encoder/layer_5/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_5/attention/self/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_5/attention/self/dropout/SelectV2" input: "bert/encoder/layer_5/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_5/attention/self/MatMul_1" input: "bert/encoder/layer_5/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_5/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_5/attention/self/transpose_3" input: "bert/encoder/layer_5/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/output/dense/kernel" } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel" input: "bert/encoder/layer_5/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/output/dense/bias" } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias" input: "bert/encoder/layer_5/attention/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_5/attention/self/Reshape_3" input: "bert/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_5/attention/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_5/attention/output/dense/MatMul" input: "bert/encoder/layer_5/attention/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_5/attention/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_5/attention/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_5/attention/output/dense/BiasAdd" input: "bert/encoder/layer_5/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_5/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/attention/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_5/attention/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_5/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_5/attention/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_5/attention/output/dropout/GreaterEqual" input: "bert/encoder/layer_5/attention/output/dropout/Mul" input: "bert/encoder/layer_5/attention/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/add" op: "AddV2" input: "bert/encoder/layer_5/attention/output/dropout/SelectV2" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/layer_normalization_11/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/layer_normalization_11/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta" } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean" op: "Mean" input: "bert/encoder/layer_5/attention/output/add" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_5/attention/output/add" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance" op: "Mean" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/Rsqrt" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_5/attention/output/add" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_1" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/intermediate/dense/kernel" } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel" input: "bert/encoder/layer_5/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/intermediate/dense/bias" } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias" input: "bert/encoder/layer_5/intermediate/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/add_1" input: "bert/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_5/intermediate/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_5/intermediate/dense/MatMul" input: "bert/encoder/layer_5/intermediate/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_5/intermediate/dense/Pow/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 3.0 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/Pow" op: "Pow" input: "bert/encoder/layer_5/intermediate/dense/BiasAdd" input: "bert/encoder/layer_5/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.044714998453855515 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_5/intermediate/dense/mul/x" input: "bert/encoder/layer_5/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/add" op: "AddV2" input: "bert/encoder/layer_5/intermediate/dense/BiasAdd" input: "bert/encoder/layer_5/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/mul_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.7978845834732056 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_5/intermediate/dense/mul_1/x" input: "bert/encoder/layer_5/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/Tanh" op: "Tanh" input: "bert/encoder/layer_5/intermediate/dense/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/add_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/add_1" op: "AddV2" input: "bert/encoder/layer_5/intermediate/dense/add_1/x" input: "bert/encoder/layer_5/intermediate/dense/Tanh" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/mul_2/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/mul_2" op: "Mul" input: "bert/encoder/layer_5/intermediate/dense/mul_2/x" input: "bert/encoder/layer_5/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/mul_3" op: "Mul" input: "bert/encoder/layer_5/intermediate/dense/BiasAdd" input: "bert/encoder/layer_5/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/output/dense/kernel" } } } node { name: "bert/encoder/layer_5/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/kernel" input: "bert/encoder/layer_5/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/output/dense/bias" } } } node { name: "bert/encoder/layer_5/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/bias" input: "bert/encoder/layer_5/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_5/intermediate/dense/mul_3" input: "bert/encoder/layer_5/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_5/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_5/output/dense/MatMul" input: "bert/encoder/layer_5/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_5/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_5/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_5/output/dense/BiasAdd" input: "bert/encoder/layer_5/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_5/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_5/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_5/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_5/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_5/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_5/output/dropout/GreaterEqual" input: "bert/encoder/layer_5/output/dropout/Mul" input: "bert/encoder/layer_5/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/add" op: "AddV2" input: "bert/encoder/layer_5/output/dropout/SelectV2" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/layer_normalization_12/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/layer_normalization_12/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/output/layer_normalization_12/gamma" } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/layer_normalization_12/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/layer_normalization_12/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/output/layer_normalization_12/beta" } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/moments/mean" op: "Mean" input: "bert/encoder/layer_5/output/add" input: "bert/encoder/layer_5/output/layer_normalization_12/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_5/output/layer_normalization_12/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_5/output/add" input: "bert/encoder/layer_5/output/layer_normalization_12/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/moments/variance" op: "Mean" input: "bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference" input: "bert/encoder/layer_5/output/layer_normalization_12/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_5/output/layer_normalization_12/moments/variance" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/Rsqrt" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_5/output/add" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_5/output/layer_normalization_12/moments/mean" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_1" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/query/kernel" } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel" input: "bert/encoder/layer_6/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/query/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/query/bias" } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias" input: "bert/encoder/layer_6/attention/self/query/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_1" input: "bert/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/query/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_6/attention/self/query/MatMul" input: "bert/encoder/layer_6/attention/self/query/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/key/kernel" } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel" input: "bert/encoder/layer_6/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/key/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/key/bias" } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias" input: "bert/encoder/layer_6/attention/self/key/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_1" input: "bert/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/key/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_6/attention/self/key/MatMul" input: "bert/encoder/layer_6/attention/self/key/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/value/kernel" } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel" input: "bert/encoder/layer_6/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/value/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/value/bias" } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias" input: "bert/encoder/layer_6/attention/self/value/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_1" input: "bert/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/value/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_6/attention/self/value/MatMul" input: "bert/encoder/layer_6/attention/self/value/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_6/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_6/attention/self/query/BiasAdd" input: "bert/encoder/layer_6/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_6/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_6/attention/self/Reshape" input: "bert/encoder/layer_6/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_6/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_6/attention/self/key/BiasAdd" input: "bert/encoder/layer_6/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_6/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_6/attention/self/Reshape_1" input: "bert/encoder/layer_6/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_6/attention/self/MatMul" op: "BatchMatMulV2" input: "bert/encoder/layer_6/attention/self/transpose" input: "bert/encoder/layer_6/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_6/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_6/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_6/attention/self/MatMul" input: "bert/encoder/layer_6/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_6/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_6/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_6/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_6/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_6/attention/self/sub" op: "Sub" input: "bert/encoder/layer_6/attention/self/sub/x" input: "bert/encoder/layer_6/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_6/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_6/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_6/attention/self/sub" input: "bert/encoder/layer_6/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_6/attention/self/add" op: "AddV2" input: "bert/encoder/layer_6/attention/self/Mul" input: "bert/encoder/layer_6/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_6/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_6/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/Mul" op: "Mul" input: "bert/encoder/layer_6/attention/self/Softmax" input: "bert/encoder/layer_6/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_6/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/attention/self/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_6/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_6/attention/self/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_6/attention/self/dropout/GreaterEqual" input: "bert/encoder/layer_6/attention/self/dropout/Mul" input: "bert/encoder/layer_6/attention/self/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_6/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_6/attention/self/value/BiasAdd" input: "bert/encoder/layer_6/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_6/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_6/attention/self/Reshape_2" input: "bert/encoder/layer_6/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_6/attention/self/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_6/attention/self/dropout/SelectV2" input: "bert/encoder/layer_6/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_6/attention/self/MatMul_1" input: "bert/encoder/layer_6/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_6/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_6/attention/self/transpose_3" input: "bert/encoder/layer_6/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/output/dense/kernel" } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel" input: "bert/encoder/layer_6/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/output/dense/bias" } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias" input: "bert/encoder/layer_6/attention/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_6/attention/self/Reshape_3" input: "bert/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_6/attention/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_6/attention/output/dense/MatMul" input: "bert/encoder/layer_6/attention/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_6/attention/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_6/attention/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_6/attention/output/dense/BiasAdd" input: "bert/encoder/layer_6/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_6/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/attention/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_6/attention/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_6/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_6/attention/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_6/attention/output/dropout/GreaterEqual" input: "bert/encoder/layer_6/attention/output/dropout/Mul" input: "bert/encoder/layer_6/attention/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/add" op: "AddV2" input: "bert/encoder/layer_6/attention/output/dropout/SelectV2" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/layer_normalization_13/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/layer_normalization_13/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta" } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean" op: "Mean" input: "bert/encoder/layer_6/attention/output/add" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_6/attention/output/add" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance" op: "Mean" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/Rsqrt" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_6/attention/output/add" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_1" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/intermediate/dense/kernel" } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel" input: "bert/encoder/layer_6/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/intermediate/dense/bias" } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias" input: "bert/encoder/layer_6/intermediate/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/add_1" input: "bert/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_6/intermediate/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_6/intermediate/dense/MatMul" input: "bert/encoder/layer_6/intermediate/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_6/intermediate/dense/Pow/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 3.0 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/Pow" op: "Pow" input: "bert/encoder/layer_6/intermediate/dense/BiasAdd" input: "bert/encoder/layer_6/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.044714998453855515 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_6/intermediate/dense/mul/x" input: "bert/encoder/layer_6/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/add" op: "AddV2" input: "bert/encoder/layer_6/intermediate/dense/BiasAdd" input: "bert/encoder/layer_6/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/mul_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.7978845834732056 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_6/intermediate/dense/mul_1/x" input: "bert/encoder/layer_6/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/Tanh" op: "Tanh" input: "bert/encoder/layer_6/intermediate/dense/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/add_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/add_1" op: "AddV2" input: "bert/encoder/layer_6/intermediate/dense/add_1/x" input: "bert/encoder/layer_6/intermediate/dense/Tanh" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/mul_2/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/mul_2" op: "Mul" input: "bert/encoder/layer_6/intermediate/dense/mul_2/x" input: "bert/encoder/layer_6/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/mul_3" op: "Mul" input: "bert/encoder/layer_6/intermediate/dense/BiasAdd" input: "bert/encoder/layer_6/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/output/dense/kernel" } } } node { name: "bert/encoder/layer_6/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/kernel" input: "bert/encoder/layer_6/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/output/dense/bias" } } } node { name: "bert/encoder/layer_6/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/bias" input: "bert/encoder/layer_6/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_6/intermediate/dense/mul_3" input: "bert/encoder/layer_6/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_6/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_6/output/dense/MatMul" input: "bert/encoder/layer_6/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_6/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_6/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_6/output/dense/BiasAdd" input: "bert/encoder/layer_6/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_6/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_6/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_6/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_6/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_6/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_6/output/dropout/GreaterEqual" input: "bert/encoder/layer_6/output/dropout/Mul" input: "bert/encoder/layer_6/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/add" op: "AddV2" input: "bert/encoder/layer_6/output/dropout/SelectV2" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/layer_normalization_14/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/layer_normalization_14/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/output/layer_normalization_14/gamma" } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/layer_normalization_14/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/layer_normalization_14/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/output/layer_normalization_14/beta" } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/moments/mean" op: "Mean" input: "bert/encoder/layer_6/output/add" input: "bert/encoder/layer_6/output/layer_normalization_14/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_6/output/layer_normalization_14/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_6/output/add" input: "bert/encoder/layer_6/output/layer_normalization_14/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/moments/variance" op: "Mean" input: "bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference" input: "bert/encoder/layer_6/output/layer_normalization_14/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_6/output/layer_normalization_14/moments/variance" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/Rsqrt" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_6/output/add" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_6/output/layer_normalization_14/moments/mean" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_1" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/query/kernel" } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel" input: "bert/encoder/layer_7/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/query/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/query/bias" } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias" input: "bert/encoder/layer_7/attention/self/query/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_1" input: "bert/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/query/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_7/attention/self/query/MatMul" input: "bert/encoder/layer_7/attention/self/query/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/key/kernel" } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel" input: "bert/encoder/layer_7/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/key/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/key/bias" } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias" input: "bert/encoder/layer_7/attention/self/key/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_1" input: "bert/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/key/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_7/attention/self/key/MatMul" input: "bert/encoder/layer_7/attention/self/key/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/value/kernel" } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel" input: "bert/encoder/layer_7/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/value/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/value/bias" } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias" input: "bert/encoder/layer_7/attention/self/value/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_1" input: "bert/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/value/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_7/attention/self/value/MatMul" input: "bert/encoder/layer_7/attention/self/value/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_7/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_7/attention/self/query/BiasAdd" input: "bert/encoder/layer_7/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_7/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_7/attention/self/Reshape" input: "bert/encoder/layer_7/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_7/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_7/attention/self/key/BiasAdd" input: "bert/encoder/layer_7/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_7/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_7/attention/self/Reshape_1" input: "bert/encoder/layer_7/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_7/attention/self/MatMul" op: "BatchMatMulV2" input: "bert/encoder/layer_7/attention/self/transpose" input: "bert/encoder/layer_7/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_7/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_7/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_7/attention/self/MatMul" input: "bert/encoder/layer_7/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_7/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_7/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_7/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_7/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_7/attention/self/sub" op: "Sub" input: "bert/encoder/layer_7/attention/self/sub/x" input: "bert/encoder/layer_7/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_7/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_7/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_7/attention/self/sub" input: "bert/encoder/layer_7/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_7/attention/self/add" op: "AddV2" input: "bert/encoder/layer_7/attention/self/Mul" input: "bert/encoder/layer_7/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_7/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_7/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/Mul" op: "Mul" input: "bert/encoder/layer_7/attention/self/Softmax" input: "bert/encoder/layer_7/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_7/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/attention/self/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_7/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_7/attention/self/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_7/attention/self/dropout/GreaterEqual" input: "bert/encoder/layer_7/attention/self/dropout/Mul" input: "bert/encoder/layer_7/attention/self/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_7/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_7/attention/self/value/BiasAdd" input: "bert/encoder/layer_7/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_7/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_7/attention/self/Reshape_2" input: "bert/encoder/layer_7/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_7/attention/self/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_7/attention/self/dropout/SelectV2" input: "bert/encoder/layer_7/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_7/attention/self/MatMul_1" input: "bert/encoder/layer_7/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_7/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_7/attention/self/transpose_3" input: "bert/encoder/layer_7/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/output/dense/kernel" } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel" input: "bert/encoder/layer_7/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/output/dense/bias" } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias" input: "bert/encoder/layer_7/attention/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_7/attention/self/Reshape_3" input: "bert/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_7/attention/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_7/attention/output/dense/MatMul" input: "bert/encoder/layer_7/attention/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_7/attention/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_7/attention/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_7/attention/output/dense/BiasAdd" input: "bert/encoder/layer_7/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_7/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/attention/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_7/attention/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_7/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_7/attention/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_7/attention/output/dropout/GreaterEqual" input: "bert/encoder/layer_7/attention/output/dropout/Mul" input: "bert/encoder/layer_7/attention/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/add" op: "AddV2" input: "bert/encoder/layer_7/attention/output/dropout/SelectV2" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/layer_normalization_15/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/layer_normalization_15/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta" } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean" op: "Mean" input: "bert/encoder/layer_7/attention/output/add" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_7/attention/output/add" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance" op: "Mean" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/Rsqrt" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_7/attention/output/add" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_1" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/intermediate/dense/kernel" } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel" input: "bert/encoder/layer_7/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/intermediate/dense/bias" } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias" input: "bert/encoder/layer_7/intermediate/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/add_1" input: "bert/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_7/intermediate/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_7/intermediate/dense/MatMul" input: "bert/encoder/layer_7/intermediate/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_7/intermediate/dense/Pow/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 3.0 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/Pow" op: "Pow" input: "bert/encoder/layer_7/intermediate/dense/BiasAdd" input: "bert/encoder/layer_7/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.044714998453855515 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_7/intermediate/dense/mul/x" input: "bert/encoder/layer_7/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/add" op: "AddV2" input: "bert/encoder/layer_7/intermediate/dense/BiasAdd" input: "bert/encoder/layer_7/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/mul_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.7978845834732056 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_7/intermediate/dense/mul_1/x" input: "bert/encoder/layer_7/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/Tanh" op: "Tanh" input: "bert/encoder/layer_7/intermediate/dense/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/add_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/add_1" op: "AddV2" input: "bert/encoder/layer_7/intermediate/dense/add_1/x" input: "bert/encoder/layer_7/intermediate/dense/Tanh" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/mul_2/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/mul_2" op: "Mul" input: "bert/encoder/layer_7/intermediate/dense/mul_2/x" input: "bert/encoder/layer_7/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/mul_3" op: "Mul" input: "bert/encoder/layer_7/intermediate/dense/BiasAdd" input: "bert/encoder/layer_7/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/output/dense/kernel" } } } node { name: "bert/encoder/layer_7/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/kernel" input: "bert/encoder/layer_7/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/output/dense/bias" } } } node { name: "bert/encoder/layer_7/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/bias" input: "bert/encoder/layer_7/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_7/intermediate/dense/mul_3" input: "bert/encoder/layer_7/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_7/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_7/output/dense/MatMul" input: "bert/encoder/layer_7/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_7/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_7/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_7/output/dense/BiasAdd" input: "bert/encoder/layer_7/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_7/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_7/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_7/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_7/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_7/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_7/output/dropout/GreaterEqual" input: "bert/encoder/layer_7/output/dropout/Mul" input: "bert/encoder/layer_7/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/add" op: "AddV2" input: "bert/encoder/layer_7/output/dropout/SelectV2" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/layer_normalization_16/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/layer_normalization_16/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/output/layer_normalization_16/gamma" } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/layer_normalization_16/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/layer_normalization_16/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/output/layer_normalization_16/beta" } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/moments/mean" op: "Mean" input: "bert/encoder/layer_7/output/add" input: "bert/encoder/layer_7/output/layer_normalization_16/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_7/output/layer_normalization_16/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_7/output/add" input: "bert/encoder/layer_7/output/layer_normalization_16/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/moments/variance" op: "Mean" input: "bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference" input: "bert/encoder/layer_7/output/layer_normalization_16/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_7/output/layer_normalization_16/moments/variance" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/Rsqrt" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_7/output/add" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_7/output/layer_normalization_16/moments/mean" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_1" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/query/kernel" } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel" input: "bert/encoder/layer_8/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/query/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/query/bias" } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias" input: "bert/encoder/layer_8/attention/self/query/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_1" input: "bert/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/query/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_8/attention/self/query/MatMul" input: "bert/encoder/layer_8/attention/self/query/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/key/kernel" } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel" input: "bert/encoder/layer_8/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/key/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/key/bias" } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias" input: "bert/encoder/layer_8/attention/self/key/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_1" input: "bert/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/key/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_8/attention/self/key/MatMul" input: "bert/encoder/layer_8/attention/self/key/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/value/kernel" } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel" input: "bert/encoder/layer_8/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/value/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/value/bias" } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias" input: "bert/encoder/layer_8/attention/self/value/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_1" input: "bert/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/value/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_8/attention/self/value/MatMul" input: "bert/encoder/layer_8/attention/self/value/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_8/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_8/attention/self/query/BiasAdd" input: "bert/encoder/layer_8/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_8/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_8/attention/self/Reshape" input: "bert/encoder/layer_8/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_8/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_8/attention/self/key/BiasAdd" input: "bert/encoder/layer_8/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_8/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_8/attention/self/Reshape_1" input: "bert/encoder/layer_8/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_8/attention/self/MatMul" op: "BatchMatMulV2" input: "bert/encoder/layer_8/attention/self/transpose" input: "bert/encoder/layer_8/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_8/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_8/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_8/attention/self/MatMul" input: "bert/encoder/layer_8/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_8/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_8/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_8/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_8/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_8/attention/self/sub" op: "Sub" input: "bert/encoder/layer_8/attention/self/sub/x" input: "bert/encoder/layer_8/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_8/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_8/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_8/attention/self/sub" input: "bert/encoder/layer_8/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_8/attention/self/add" op: "AddV2" input: "bert/encoder/layer_8/attention/self/Mul" input: "bert/encoder/layer_8/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_8/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_8/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/Mul" op: "Mul" input: "bert/encoder/layer_8/attention/self/Softmax" input: "bert/encoder/layer_8/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_8/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/attention/self/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_8/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_8/attention/self/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_8/attention/self/dropout/GreaterEqual" input: "bert/encoder/layer_8/attention/self/dropout/Mul" input: "bert/encoder/layer_8/attention/self/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_8/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_8/attention/self/value/BiasAdd" input: "bert/encoder/layer_8/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_8/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_8/attention/self/Reshape_2" input: "bert/encoder/layer_8/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_8/attention/self/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_8/attention/self/dropout/SelectV2" input: "bert/encoder/layer_8/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_8/attention/self/MatMul_1" input: "bert/encoder/layer_8/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_8/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_8/attention/self/transpose_3" input: "bert/encoder/layer_8/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/output/dense/kernel" } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel" input: "bert/encoder/layer_8/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/output/dense/bias" } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias" input: "bert/encoder/layer_8/attention/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_8/attention/self/Reshape_3" input: "bert/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_8/attention/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_8/attention/output/dense/MatMul" input: "bert/encoder/layer_8/attention/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_8/attention/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_8/attention/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_8/attention/output/dense/BiasAdd" input: "bert/encoder/layer_8/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_8/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/attention/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_8/attention/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_8/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_8/attention/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_8/attention/output/dropout/GreaterEqual" input: "bert/encoder/layer_8/attention/output/dropout/Mul" input: "bert/encoder/layer_8/attention/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/add" op: "AddV2" input: "bert/encoder/layer_8/attention/output/dropout/SelectV2" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/layer_normalization_17/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/layer_normalization_17/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta" } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean" op: "Mean" input: "bert/encoder/layer_8/attention/output/add" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_8/attention/output/add" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance" op: "Mean" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/Rsqrt" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_8/attention/output/add" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_1" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/intermediate/dense/kernel" } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel" input: "bert/encoder/layer_8/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/intermediate/dense/bias" } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias" input: "bert/encoder/layer_8/intermediate/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/add_1" input: "bert/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_8/intermediate/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_8/intermediate/dense/MatMul" input: "bert/encoder/layer_8/intermediate/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_8/intermediate/dense/Pow/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 3.0 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/Pow" op: "Pow" input: "bert/encoder/layer_8/intermediate/dense/BiasAdd" input: "bert/encoder/layer_8/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.044714998453855515 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_8/intermediate/dense/mul/x" input: "bert/encoder/layer_8/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/add" op: "AddV2" input: "bert/encoder/layer_8/intermediate/dense/BiasAdd" input: "bert/encoder/layer_8/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/mul_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.7978845834732056 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_8/intermediate/dense/mul_1/x" input: "bert/encoder/layer_8/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/Tanh" op: "Tanh" input: "bert/encoder/layer_8/intermediate/dense/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/add_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/add_1" op: "AddV2" input: "bert/encoder/layer_8/intermediate/dense/add_1/x" input: "bert/encoder/layer_8/intermediate/dense/Tanh" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/mul_2/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/mul_2" op: "Mul" input: "bert/encoder/layer_8/intermediate/dense/mul_2/x" input: "bert/encoder/layer_8/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/mul_3" op: "Mul" input: "bert/encoder/layer_8/intermediate/dense/BiasAdd" input: "bert/encoder/layer_8/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/output/dense/kernel" } } } node { name: "bert/encoder/layer_8/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/kernel" input: "bert/encoder/layer_8/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/output/dense/bias" } } } node { name: "bert/encoder/layer_8/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/bias" input: "bert/encoder/layer_8/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_8/intermediate/dense/mul_3" input: "bert/encoder/layer_8/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_8/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_8/output/dense/MatMul" input: "bert/encoder/layer_8/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_8/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_8/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_8/output/dense/BiasAdd" input: "bert/encoder/layer_8/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_8/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_8/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_8/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_8/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_8/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_8/output/dropout/GreaterEqual" input: "bert/encoder/layer_8/output/dropout/Mul" input: "bert/encoder/layer_8/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/add" op: "AddV2" input: "bert/encoder/layer_8/output/dropout/SelectV2" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/layer_normalization_18/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/layer_normalization_18/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/output/layer_normalization_18/gamma" } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/layer_normalization_18/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/layer_normalization_18/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/output/layer_normalization_18/beta" } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/moments/mean" op: "Mean" input: "bert/encoder/layer_8/output/add" input: "bert/encoder/layer_8/output/layer_normalization_18/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_8/output/layer_normalization_18/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_8/output/add" input: "bert/encoder/layer_8/output/layer_normalization_18/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/moments/variance" op: "Mean" input: "bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference" input: "bert/encoder/layer_8/output/layer_normalization_18/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_8/output/layer_normalization_18/moments/variance" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/Rsqrt" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_8/output/add" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_8/output/layer_normalization_18/moments/mean" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_1" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/query/kernel" } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel" input: "bert/encoder/layer_9/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/query/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/query/bias" } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias" input: "bert/encoder/layer_9/attention/self/query/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_1" input: "bert/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/query/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_9/attention/self/query/MatMul" input: "bert/encoder/layer_9/attention/self/query/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/key/kernel" } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel" input: "bert/encoder/layer_9/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/key/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/key/bias" } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias" input: "bert/encoder/layer_9/attention/self/key/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_1" input: "bert/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/key/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_9/attention/self/key/MatMul" input: "bert/encoder/layer_9/attention/self/key/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/value/kernel" } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel" input: "bert/encoder/layer_9/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/value/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/value/bias" } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias" input: "bert/encoder/layer_9/attention/self/value/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_1" input: "bert/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/value/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_9/attention/self/value/MatMul" input: "bert/encoder/layer_9/attention/self/value/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_9/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_9/attention/self/query/BiasAdd" input: "bert/encoder/layer_9/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_9/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_9/attention/self/Reshape" input: "bert/encoder/layer_9/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_9/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_9/attention/self/key/BiasAdd" input: "bert/encoder/layer_9/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_9/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_9/attention/self/Reshape_1" input: "bert/encoder/layer_9/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_9/attention/self/MatMul" op: "BatchMatMulV2" input: "bert/encoder/layer_9/attention/self/transpose" input: "bert/encoder/layer_9/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_9/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_9/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_9/attention/self/MatMul" input: "bert/encoder/layer_9/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_9/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_9/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_9/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_9/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_9/attention/self/sub" op: "Sub" input: "bert/encoder/layer_9/attention/self/sub/x" input: "bert/encoder/layer_9/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_9/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_9/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_9/attention/self/sub" input: "bert/encoder/layer_9/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_9/attention/self/add" op: "AddV2" input: "bert/encoder/layer_9/attention/self/Mul" input: "bert/encoder/layer_9/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_9/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_9/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/Mul" op: "Mul" input: "bert/encoder/layer_9/attention/self/Softmax" input: "bert/encoder/layer_9/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_9/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/attention/self/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_9/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_9/attention/self/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_9/attention/self/dropout/GreaterEqual" input: "bert/encoder/layer_9/attention/self/dropout/Mul" input: "bert/encoder/layer_9/attention/self/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_9/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_9/attention/self/value/BiasAdd" input: "bert/encoder/layer_9/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_9/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_9/attention/self/Reshape_2" input: "bert/encoder/layer_9/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_9/attention/self/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_9/attention/self/dropout/SelectV2" input: "bert/encoder/layer_9/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_9/attention/self/MatMul_1" input: "bert/encoder/layer_9/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_9/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_9/attention/self/transpose_3" input: "bert/encoder/layer_9/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/output/dense/kernel" } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel" input: "bert/encoder/layer_9/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/output/dense/bias" } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias" input: "bert/encoder/layer_9/attention/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_9/attention/self/Reshape_3" input: "bert/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_9/attention/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_9/attention/output/dense/MatMul" input: "bert/encoder/layer_9/attention/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_9/attention/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_9/attention/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_9/attention/output/dense/BiasAdd" input: "bert/encoder/layer_9/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_9/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/attention/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_9/attention/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_9/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_9/attention/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_9/attention/output/dropout/GreaterEqual" input: "bert/encoder/layer_9/attention/output/dropout/Mul" input: "bert/encoder/layer_9/attention/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/add" op: "AddV2" input: "bert/encoder/layer_9/attention/output/dropout/SelectV2" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/layer_normalization_19/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/layer_normalization_19/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta" } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean" op: "Mean" input: "bert/encoder/layer_9/attention/output/add" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_9/attention/output/add" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance" op: "Mean" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/Rsqrt" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_9/attention/output/add" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_1" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/intermediate/dense/kernel" } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel" input: "bert/encoder/layer_9/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/intermediate/dense/bias" } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias" input: "bert/encoder/layer_9/intermediate/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/add_1" input: "bert/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_9/intermediate/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_9/intermediate/dense/MatMul" input: "bert/encoder/layer_9/intermediate/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_9/intermediate/dense/Pow/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 3.0 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/Pow" op: "Pow" input: "bert/encoder/layer_9/intermediate/dense/BiasAdd" input: "bert/encoder/layer_9/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.044714998453855515 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_9/intermediate/dense/mul/x" input: "bert/encoder/layer_9/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/add" op: "AddV2" input: "bert/encoder/layer_9/intermediate/dense/BiasAdd" input: "bert/encoder/layer_9/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/mul_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.7978845834732056 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_9/intermediate/dense/mul_1/x" input: "bert/encoder/layer_9/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/Tanh" op: "Tanh" input: "bert/encoder/layer_9/intermediate/dense/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/add_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/add_1" op: "AddV2" input: "bert/encoder/layer_9/intermediate/dense/add_1/x" input: "bert/encoder/layer_9/intermediate/dense/Tanh" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/mul_2/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/mul_2" op: "Mul" input: "bert/encoder/layer_9/intermediate/dense/mul_2/x" input: "bert/encoder/layer_9/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/mul_3" op: "Mul" input: "bert/encoder/layer_9/intermediate/dense/BiasAdd" input: "bert/encoder/layer_9/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/output/dense/kernel" } } } node { name: "bert/encoder/layer_9/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/kernel" input: "bert/encoder/layer_9/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/output/dense/bias" } } } node { name: "bert/encoder/layer_9/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/bias" input: "bert/encoder/layer_9/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_9/intermediate/dense/mul_3" input: "bert/encoder/layer_9/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_9/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_9/output/dense/MatMul" input: "bert/encoder/layer_9/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_9/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_9/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_9/output/dense/BiasAdd" input: "bert/encoder/layer_9/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_9/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_9/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_9/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_9/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_9/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_9/output/dropout/GreaterEqual" input: "bert/encoder/layer_9/output/dropout/Mul" input: "bert/encoder/layer_9/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/add" op: "AddV2" input: "bert/encoder/layer_9/output/dropout/SelectV2" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/layer_normalization_20/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/layer_normalization_20/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/output/layer_normalization_20/gamma" } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/layer_normalization_20/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/layer_normalization_20/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/output/layer_normalization_20/beta" } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/moments/mean" op: "Mean" input: "bert/encoder/layer_9/output/add" input: "bert/encoder/layer_9/output/layer_normalization_20/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_9/output/layer_normalization_20/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_9/output/add" input: "bert/encoder/layer_9/output/layer_normalization_20/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/moments/variance" op: "Mean" input: "bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference" input: "bert/encoder/layer_9/output/layer_normalization_20/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_9/output/layer_normalization_20/moments/variance" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/Rsqrt" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_9/output/add" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_9/output/layer_normalization_20/moments/mean" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_1" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/query/kernel" } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel" input: "bert/encoder/layer_10/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/query/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/query/bias" } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias" input: "bert/encoder/layer_10/attention/self/query/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_1" input: "bert/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/query/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_10/attention/self/query/MatMul" input: "bert/encoder/layer_10/attention/self/query/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/key/kernel" } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel" input: "bert/encoder/layer_10/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/key/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/key/bias" } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias" input: "bert/encoder/layer_10/attention/self/key/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_1" input: "bert/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/key/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_10/attention/self/key/MatMul" input: "bert/encoder/layer_10/attention/self/key/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/value/kernel" } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel" input: "bert/encoder/layer_10/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/value/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/value/bias" } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias" input: "bert/encoder/layer_10/attention/self/value/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_1" input: "bert/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/value/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_10/attention/self/value/MatMul" input: "bert/encoder/layer_10/attention/self/value/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_10/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_10/attention/self/query/BiasAdd" input: "bert/encoder/layer_10/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_10/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_10/attention/self/Reshape" input: "bert/encoder/layer_10/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_10/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_10/attention/self/key/BiasAdd" input: "bert/encoder/layer_10/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_10/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_10/attention/self/Reshape_1" input: "bert/encoder/layer_10/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_10/attention/self/MatMul" op: "BatchMatMulV2" input: "bert/encoder/layer_10/attention/self/transpose" input: "bert/encoder/layer_10/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_10/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_10/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_10/attention/self/MatMul" input: "bert/encoder/layer_10/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_10/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_10/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_10/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_10/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_10/attention/self/sub" op: "Sub" input: "bert/encoder/layer_10/attention/self/sub/x" input: "bert/encoder/layer_10/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_10/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_10/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_10/attention/self/sub" input: "bert/encoder/layer_10/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_10/attention/self/add" op: "AddV2" input: "bert/encoder/layer_10/attention/self/Mul" input: "bert/encoder/layer_10/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_10/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_10/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/Mul" op: "Mul" input: "bert/encoder/layer_10/attention/self/Softmax" input: "bert/encoder/layer_10/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_10/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/attention/self/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_10/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_10/attention/self/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_10/attention/self/dropout/GreaterEqual" input: "bert/encoder/layer_10/attention/self/dropout/Mul" input: "bert/encoder/layer_10/attention/self/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_10/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_10/attention/self/value/BiasAdd" input: "bert/encoder/layer_10/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_10/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_10/attention/self/Reshape_2" input: "bert/encoder/layer_10/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_10/attention/self/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_10/attention/self/dropout/SelectV2" input: "bert/encoder/layer_10/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_10/attention/self/MatMul_1" input: "bert/encoder/layer_10/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_10/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_10/attention/self/transpose_3" input: "bert/encoder/layer_10/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/output/dense/kernel" } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel" input: "bert/encoder/layer_10/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/output/dense/bias" } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias" input: "bert/encoder/layer_10/attention/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_10/attention/self/Reshape_3" input: "bert/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_10/attention/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_10/attention/output/dense/MatMul" input: "bert/encoder/layer_10/attention/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_10/attention/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_10/attention/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_10/attention/output/dense/BiasAdd" input: "bert/encoder/layer_10/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_10/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/attention/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_10/attention/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_10/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_10/attention/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_10/attention/output/dropout/GreaterEqual" input: "bert/encoder/layer_10/attention/output/dropout/Mul" input: "bert/encoder/layer_10/attention/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/add" op: "AddV2" input: "bert/encoder/layer_10/attention/output/dropout/SelectV2" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/layer_normalization_21/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/layer_normalization_21/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta" } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean" op: "Mean" input: "bert/encoder/layer_10/attention/output/add" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_10/attention/output/add" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance" op: "Mean" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/Rsqrt" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_10/attention/output/add" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_1" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/intermediate/dense/kernel" } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel" input: "bert/encoder/layer_10/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/intermediate/dense/bias" } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias" input: "bert/encoder/layer_10/intermediate/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/add_1" input: "bert/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_10/intermediate/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_10/intermediate/dense/MatMul" input: "bert/encoder/layer_10/intermediate/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_10/intermediate/dense/Pow/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 3.0 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/Pow" op: "Pow" input: "bert/encoder/layer_10/intermediate/dense/BiasAdd" input: "bert/encoder/layer_10/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.044714998453855515 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_10/intermediate/dense/mul/x" input: "bert/encoder/layer_10/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/add" op: "AddV2" input: "bert/encoder/layer_10/intermediate/dense/BiasAdd" input: "bert/encoder/layer_10/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/mul_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.7978845834732056 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_10/intermediate/dense/mul_1/x" input: "bert/encoder/layer_10/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/Tanh" op: "Tanh" input: "bert/encoder/layer_10/intermediate/dense/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/add_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/add_1" op: "AddV2" input: "bert/encoder/layer_10/intermediate/dense/add_1/x" input: "bert/encoder/layer_10/intermediate/dense/Tanh" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/mul_2/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/mul_2" op: "Mul" input: "bert/encoder/layer_10/intermediate/dense/mul_2/x" input: "bert/encoder/layer_10/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/mul_3" op: "Mul" input: "bert/encoder/layer_10/intermediate/dense/BiasAdd" input: "bert/encoder/layer_10/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/output/dense/kernel" } } } node { name: "bert/encoder/layer_10/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/kernel" input: "bert/encoder/layer_10/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/output/dense/bias" } } } node { name: "bert/encoder/layer_10/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/bias" input: "bert/encoder/layer_10/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_10/intermediate/dense/mul_3" input: "bert/encoder/layer_10/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_10/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_10/output/dense/MatMul" input: "bert/encoder/layer_10/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_10/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_10/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_10/output/dense/BiasAdd" input: "bert/encoder/layer_10/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_10/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_10/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_10/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_10/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_10/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_10/output/dropout/GreaterEqual" input: "bert/encoder/layer_10/output/dropout/Mul" input: "bert/encoder/layer_10/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/add" op: "AddV2" input: "bert/encoder/layer_10/output/dropout/SelectV2" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/layer_normalization_22/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/layer_normalization_22/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/output/layer_normalization_22/gamma" } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/layer_normalization_22/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/layer_normalization_22/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/output/layer_normalization_22/beta" } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/moments/mean" op: "Mean" input: "bert/encoder/layer_10/output/add" input: "bert/encoder/layer_10/output/layer_normalization_22/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_10/output/layer_normalization_22/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_10/output/add" input: "bert/encoder/layer_10/output/layer_normalization_22/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/moments/variance" op: "Mean" input: "bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference" input: "bert/encoder/layer_10/output/layer_normalization_22/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_10/output/layer_normalization_22/moments/variance" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/Rsqrt" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_10/output/add" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_10/output/layer_normalization_22/moments/mean" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_1" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/query/kernel" } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel" input: "bert/encoder/layer_11/attention/self/query/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/query/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/query/bias" } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias" input: "bert/encoder/layer_11/attention/self/query/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/query/MatMul" op: "MatMul" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_1" input: "bert/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/query/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/query/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_11/attention/self/query/MatMul" input: "bert/encoder/layer_11/attention/self/query/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/key/kernel" } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel" input: "bert/encoder/layer_11/attention/self/key/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/key/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/key/bias" } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias" input: "bert/encoder/layer_11/attention/self/key/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/key/MatMul" op: "MatMul" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_1" input: "bert/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/key/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/key/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_11/attention/self/key/MatMul" input: "bert/encoder/layer_11/attention/self/key/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/value/kernel" } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel" input: "bert/encoder/layer_11/attention/self/value/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/value/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/value/bias" } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias" input: "bert/encoder/layer_11/attention/self/value/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/value/MatMul" op: "MatMul" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_1" input: "bert/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/value/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/value/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_11/attention/self/value/MatMul" input: "bert/encoder/layer_11/attention/self/value/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_11/attention/self/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/Reshape" op: "Reshape" input: "bert/encoder/layer_11/attention/self/query/BiasAdd" input: "bert/encoder/layer_11/attention/self/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_11/attention/self/transpose/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/transpose" op: "Transpose" input: "bert/encoder/layer_11/attention/self/Reshape" input: "bert/encoder/layer_11/attention/self/transpose/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_11/attention/self/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/Reshape_1" op: "Reshape" input: "bert/encoder/layer_11/attention/self/key/BiasAdd" input: "bert/encoder/layer_11/attention/self/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_11/attention/self/transpose_1/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/transpose_1" op: "Transpose" input: "bert/encoder/layer_11/attention/self/Reshape_1" input: "bert/encoder/layer_11/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_11/attention/self/MatMul" op: "BatchMatMulV2" input: "bert/encoder/layer_11/attention/self/transpose" input: "bert/encoder/layer_11/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "bert/encoder/layer_11/attention/self/Mul/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.125 } } } } node { name: "bert/encoder/layer_11/attention/self/Mul" op: "Mul" input: "bert/encoder/layer_11/attention/self/MatMul" input: "bert/encoder/layer_11/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_11/attention/self/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_11/attention/self/ExpandDims" op: "ExpandDims" input: "bert/encoder/mul" input: "bert/encoder/layer_11/attention/self/ExpandDims/dim" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_11/attention/self/sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_11/attention/self/sub" op: "Sub" input: "bert/encoder/layer_11/attention/self/sub/x" input: "bert/encoder/layer_11/attention/self/ExpandDims" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_11/attention/self/mul_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: -10000.0 } } } } node { name: "bert/encoder/layer_11/attention/self/mul_1" op: "Mul" input: "bert/encoder/layer_11/attention/self/sub" input: "bert/encoder/layer_11/attention/self/mul_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_11/attention/self/add" op: "AddV2" input: "bert/encoder/layer_11/attention/self/Mul" input: "bert/encoder/layer_11/attention/self/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_11/attention/self/Softmax" op: "Softmax" input: "bert/encoder/layer_11/attention/self/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/Mul" op: "Mul" input: "bert/encoder/layer_11/attention/self/Softmax" input: "bert/encoder/layer_11/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_11/attention/self/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/attention/self/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_11/attention/self/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_11/attention/self/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_11/attention/self/dropout/GreaterEqual" input: "bert/encoder/layer_11/attention/self/dropout/Mul" input: "bert/encoder/layer_11/attention/self/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "bert/encoder/layer_11/attention/self/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/Reshape_2" op: "Reshape" input: "bert/encoder/layer_11/attention/self/value/BiasAdd" input: "bert/encoder/layer_11/attention/self/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_11/attention/self/transpose_2/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/transpose_2" op: "Transpose" input: "bert/encoder/layer_11/attention/self/Reshape_2" input: "bert/encoder/layer_11/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_11/attention/self/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_11/attention/self/dropout/SelectV2" input: "bert/encoder/layer_11/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/transpose_3/perm" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\002\000\000\000\001\000\000\000\003\000\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/transpose_3" op: "Transpose" input: "bert/encoder/layer_11/attention/self/MatMul_1" input: "bert/encoder/layer_11/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "bert/encoder/layer_11/attention/self/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/Reshape_3" op: "Reshape" input: "bert/encoder/layer_11/attention/self/transpose_3" input: "bert/encoder/layer_11/attention/self/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/output/dense/kernel" } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel" input: "bert/encoder/layer_11/attention/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/output/dense/bias" } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias" input: "bert/encoder/layer_11/attention/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_11/attention/self/Reshape_3" input: "bert/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_11/attention/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_11/attention/output/dense/MatMul" input: "bert/encoder/layer_11/attention/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_11/attention/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_11/attention/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_11/attention/output/dense/BiasAdd" input: "bert/encoder/layer_11/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_11/attention/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/attention/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_11/attention/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_11/attention/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_11/attention/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_11/attention/output/dropout/GreaterEqual" input: "bert/encoder/layer_11/attention/output/dropout/Mul" input: "bert/encoder/layer_11/attention/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/add" op: "AddV2" input: "bert/encoder/layer_11/attention/output/dropout/SelectV2" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/layer_normalization_23/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/layer_normalization_23/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta" } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean" op: "Mean" input: "bert/encoder/layer_11/attention/output/add" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_11/attention/output/add" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance" op: "Mean" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/Rsqrt" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_11/attention/output/add" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_1" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/intermediate/dense/kernel" } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel" input: "bert/encoder/layer_11/intermediate/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/intermediate/dense/bias/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/intermediate/dense/bias/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/intermediate/dense/bias" } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias" input: "bert/encoder/layer_11/intermediate/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/intermediate/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/add_1" input: "bert/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_11/intermediate/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/intermediate/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_11/intermediate/dense/MatMul" input: "bert/encoder/layer_11/intermediate/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_11/intermediate/dense/Pow/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 3.0 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/Pow" op: "Pow" input: "bert/encoder/layer_11/intermediate/dense/BiasAdd" input: "bert/encoder/layer_11/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.044714998453855515 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/mul" op: "Mul" input: "bert/encoder/layer_11/intermediate/dense/mul/x" input: "bert/encoder/layer_11/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/add" op: "AddV2" input: "bert/encoder/layer_11/intermediate/dense/BiasAdd" input: "bert/encoder/layer_11/intermediate/dense/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/mul_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.7978845834732056 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/mul_1" op: "Mul" input: "bert/encoder/layer_11/intermediate/dense/mul_1/x" input: "bert/encoder/layer_11/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/Tanh" op: "Tanh" input: "bert/encoder/layer_11/intermediate/dense/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/add_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/add_1" op: "AddV2" input: "bert/encoder/layer_11/intermediate/dense/add_1/x" input: "bert/encoder/layer_11/intermediate/dense/Tanh" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/mul_2/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.5 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/mul_2" op: "Mul" input: "bert/encoder/layer_11/intermediate/dense/mul_2/x" input: "bert/encoder/layer_11/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/mul_3" op: "Mul" input: "bert/encoder/layer_11/intermediate/dense/BiasAdd" input: "bert/encoder/layer_11/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/mul" input: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/output/dense/kernel" } } } node { name: "bert/encoder/layer_11/output/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/kernel" input: "bert/encoder/layer_11/output/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/output/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/output/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/output/dense/bias" } } } node { name: "bert/encoder/layer_11/output/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/output/dense/bias/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/bias" input: "bert/encoder/layer_11/output/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/output/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/output/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/output/dense/MatMul" op: "MatMul" input: "bert/encoder/layer_11/intermediate/dense/mul_3" input: "bert/encoder/layer_11/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/encoder/layer_11/output/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/output/dense/BiasAdd" op: "BiasAdd" input: "bert/encoder/layer_11/output/dense/MatMul" input: "bert/encoder/layer_11/output/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/encoder/layer_11/output/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "bert/encoder/layer_11/output/dropout/Mul" op: "Mul" input: "bert/encoder/layer_11/output/dense/BiasAdd" input: "bert/encoder/layer_11/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/output/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "bert/encoder/layer_11/output/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/encoder/layer_11/output/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "bert/encoder/layer_11/output/dropout/GreaterEqual" op: "GreaterEqual" input: "bert/encoder/layer_11/output/dropout/random_uniform/RandomUniform" input: "bert/encoder/layer_11/output/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/dropout/SelectV2" op: "SelectV2" input: "bert/encoder/layer_11/output/dropout/GreaterEqual" input: "bert/encoder/layer_11/output/dropout/Mul" input: "bert/encoder/layer_11/output/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/add" op: "AddV2" input: "bert/encoder/layer_11/output/dropout/SelectV2" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/gamma/Initializer/ones" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/layer_normalization_24/gamma" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 1.0 } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/gamma" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/layer_normalization_24/gamma" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/output/layer_normalization_24/gamma" } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/gamma/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/gamma/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/Initializer/ones" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/gamma/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/beta/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/layer_normalization_24/beta" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/beta" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/layer_normalization_24/beta" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/output/layer_normalization_24/beta" } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/beta/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/beta/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/beta/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/moments/mean/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/moments/mean" op: "Mean" input: "bert/encoder/layer_11/output/add" input: "bert/encoder/layer_11/output/layer_normalization_24/moments/mean/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/moments/StopGradient" op: "StopGradient" input: "bert/encoder/layer_11/output/layer_normalization_24/moments/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference" op: "SquaredDifference" input: "bert/encoder/layer_11/output/add" input: "bert/encoder/layer_11/output/layer_normalization_24/moments/StopGradient" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/moments/variance/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/moments/variance" op: "Mean" input: "bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference" input: "bert/encoder/layer_11/output/layer_normalization_24/moments/variance/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999960041972e-13 } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add" op: "AddV2" input: "bert/encoder/layer_11/output/layer_normalization_24/moments/variance" input: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/Rsqrt" op: "Rsqrt" input: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul" op: "Mul" input: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/Rsqrt" input: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_1" op: "Mul" input: "bert/encoder/layer_11/output/add" input: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2" op: "Mul" input: "bert/encoder/layer_11/output/layer_normalization_24/moments/mean" input: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub" op: "Sub" input: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add_1" op: "AddV2" input: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_1" input: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_2/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_2" op: "Reshape" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_1" input: "bert/encoder/Reshape_2/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_3/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_3" op: "Reshape" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_1" input: "bert/encoder/Reshape_3/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_4/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_4" op: "Reshape" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_1" input: "bert/encoder/Reshape_4/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_5/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_5" op: "Reshape" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_1" input: "bert/encoder/Reshape_5/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_6/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_6" op: "Reshape" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_1" input: "bert/encoder/Reshape_6/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_7/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_7" op: "Reshape" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_1" input: "bert/encoder/Reshape_7/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_8/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_8" op: "Reshape" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_1" input: "bert/encoder/Reshape_8/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_9/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_9" op: "Reshape" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_1" input: "bert/encoder/Reshape_9/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_10/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_10" op: "Reshape" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_1" input: "bert/encoder/Reshape_10/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_11/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_11" op: "Reshape" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_1" input: "bert/encoder/Reshape_11/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_12/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_12" op: "Reshape" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_1" input: "bert/encoder/Reshape_12/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/encoder/Reshape_13/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "bert/encoder/Reshape_13" op: "Reshape" input: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add_1" input: "bert/encoder/Reshape_13/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "bert/pooler/strided_slice/stack" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\000\000\000\000\000\000\000\000\000\000\000\000" } } } } node { name: "bert/pooler/strided_slice/stack_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\000\000\000\000\001\000\000\000\000\000\000\000" } } } } node { name: "bert/pooler/strided_slice/stack_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\001\000\000\000\001\000\000\000\001\000\000\000" } } } } node { name: "bert/pooler/strided_slice" op: "StridedSlice" input: "bert/encoder/Reshape_13" input: "bert/pooler/strided_slice/stack" input: "bert/pooler/strided_slice/stack_1" input: "bert/pooler/strided_slice/stack_2" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 768 } } } } } attr { key: "begin_mask" value { i: 5 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 5 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "bert/pooler/Squeeze" op: "Squeeze" input: "bert/pooler/strided_slice" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } attr { key: "squeeze_dims" value { list { i: 1 } } } } node { name: "bert/pooler/dense/kernel/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/pooler/dense/kernel/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/pooler/dense/kernel/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "bert/pooler/dense/kernel/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "bert/pooler/dense/kernel/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "bert/pooler/dense/kernel/Initializer/truncated_normal/mul" op: "Mul" input: "bert/pooler/dense/kernel/Initializer/truncated_normal/TruncatedNormal" input: "bert/pooler/dense/kernel/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/pooler/dense/kernel/Initializer/truncated_normal" op: "AddV2" input: "bert/pooler/dense/kernel/Initializer/truncated_normal/mul" input: "bert/pooler/dense/kernel/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "bert/pooler/dense/kernel" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/pooler/dense/kernel" } } } node { name: "bert/pooler/dense/kernel/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/pooler/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/pooler/dense/kernel/Assign" op: "AssignVariableOp" input: "bert/pooler/dense/kernel" input: "bert/pooler/dense/kernel/Initializer/truncated_normal" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/pooler/dense/kernel/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/pooler/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/pooler/dense/bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/pooler/dense/bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/pooler/dense/bias" } } } node { name: "bert/pooler/dense/bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/pooler/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/pooler/dense/bias/Assign" op: "AssignVariableOp" input: "bert/pooler/dense/bias" input: "bert/pooler/dense/bias/Initializer/zeros" attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/pooler/dense/bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/pooler/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/pooler/dense/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "bert/pooler/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/pooler/dense/MatMul" op: "MatMul" input: "bert/pooler/Squeeze" input: "bert/pooler/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "bert/pooler/dense/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "bert/pooler/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/pooler/dense/BiasAdd" op: "BiasAdd" input: "bert/pooler/dense/MatMul" input: "bert/pooler/dense/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "bert/pooler/dense/Tanh" op: "Tanh" input: "bert/pooler/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } } node { name: "output_weights/Initializer/truncated_normal/shape" op: "Const" attr { key: "_class" value { list { s: "loc:@output_weights" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\003\000\000\000\000\003\000\000" } } } } node { name: "output_weights/Initializer/truncated_normal/mean" op: "Const" attr { key: "_class" value { list { s: "loc:@output_weights" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "output_weights/Initializer/truncated_normal/stddev" op: "Const" attr { key: "_class" value { list { s: "loc:@output_weights" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.019999999552965164 } } } } node { name: "output_weights/Initializer/truncated_normal/TruncatedNormal" op: "TruncatedNormal" input: "output_weights/Initializer/truncated_normal/shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_class" value { list { s: "loc:@output_weights" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "output_weights/Initializer/truncated_normal/mul" op: "Mul" input: "output_weights/Initializer/truncated_normal/TruncatedNormal" input: "output_weights/Initializer/truncated_normal/stddev" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@output_weights" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "output_weights/Initializer/truncated_normal" op: "AddV2" input: "output_weights/Initializer/truncated_normal/mul" input: "output_weights/Initializer/truncated_normal/mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@output_weights" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "output_weights" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@output_weights" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "output_weights" } } } node { name: "output_weights/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "output_weights" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "output_weights/Assign" op: "AssignVariableOp" input: "output_weights" input: "output_weights/Initializer/truncated_normal" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "output_weights/Read/ReadVariableOp" op: "ReadVariableOp" input: "output_weights" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "output_bias/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@output_bias" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 3 } } float_val: 0.0 } } } } node { name: "output_bias" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@output_bias" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3 } } } } attr { key: "shared_name" value { s: "output_bias" } } } node { name: "output_bias/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "output_bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "output_bias/Assign" op: "AssignVariableOp" input: "output_bias" input: "output_bias/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "output_bias/Read/ReadVariableOp" op: "ReadVariableOp" input: "output_bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "loss/dropout/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.1111111640930176 } } } } node { name: "loss/dropout/Mul" op: "Mul" input: "bert/pooler/dense/Tanh" input: "loss/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } } node { name: "loss/dropout/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\020\000\000\000\000\003\000\000" } } } } node { name: "loss/dropout/random_uniform/RandomUniform" op: "RandomUniform" input: "loss/dropout/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "seed" value { i: 0 } } attr { key: "seed2" value { i: 0 } } } node { name: "loss/dropout/GreaterEqual/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "loss/dropout/GreaterEqual" op: "GreaterEqual" input: "loss/dropout/random_uniform/RandomUniform" input: "loss/dropout/GreaterEqual/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } } node { name: "loss/dropout/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "loss/dropout/SelectV2" op: "SelectV2" input: "loss/dropout/GreaterEqual" input: "loss/dropout/Mul" input: "loss/dropout/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } } node { name: "loss/MatMul/ReadVariableOp" op: "ReadVariableOp" input: "output_weights" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "loss/MatMul" op: "MatMul" input: "loss/dropout/SelectV2" input: "loss/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 3 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "loss/BiasAdd/ReadVariableOp" op: "ReadVariableOp" input: "output_bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "loss/BiasAdd" op: "BiasAdd" input: "loss/MatMul" input: "loss/BiasAdd/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 3 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "loss/Softmax" op: "Softmax" input: "loss/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 3 } } } } } } node { name: "loss/LogSoftmax" op: "LogSoftmax" input: "loss/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 3 } } } } } } node { name: "loss/one_hot/on_value" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "loss/one_hot/off_value" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "loss/one_hot/depth" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 3 } } } } node { name: "loss/one_hot" op: "OneHot" input: "IteratorGetNext:3" input: "loss/one_hot/depth" input: "loss/one_hot/on_value" input: "loss/one_hot/off_value" attr { key: "T" value { type: DT_FLOAT } } attr { key: "TI" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 3 } } } } } attr { key: "axis" value { i: -1 } } } node { name: "loss/mul" op: "Mul" input: "loss/one_hot" input: "loss/LogSoftmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 3 } } } } } } node { name: "loss/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "loss/Sum" op: "Sum" input: "loss/mul" input: "loss/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "loss/Neg" op: "Neg" input: "loss/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } } } } } } node { name: "loss/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "loss/Mean" op: "Mean" input: "loss/Neg" input: "loss/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "checkpoint_initializer/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/embeddings/position_embeddings" } } } } node { name: "checkpoint_initializer/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer" op: "RestoreV2" input: "checkpoint_initializer/prefix" input: "checkpoint_initializer/tensor_names" input: "checkpoint_initializer/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity" op: "Identity" input: "checkpoint_initializer" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp" op: "AssignVariableOp" input: "bert/embeddings/position_embeddings" input: "Identity" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_1/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_1/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/embeddings/token_type_embeddings" } } } } node { name: "checkpoint_initializer_1/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_1" op: "RestoreV2" input: "checkpoint_initializer_1/prefix" input: "checkpoint_initializer_1/tensor_names" input: "checkpoint_initializer_1/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_1" op: "Identity" input: "checkpoint_initializer_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_1" op: "AssignVariableOp" input: "bert/embeddings/token_type_embeddings" input: "Identity_1" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_2/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_2/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/embeddings/word_embeddings" } } } } node { name: "checkpoint_initializer_2/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_2" op: "RestoreV2" input: "checkpoint_initializer_2/prefix" input: "checkpoint_initializer_2/tensor_names" input: "checkpoint_initializer_2/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_2" op: "Identity" input: "checkpoint_initializer_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_2" op: "AssignVariableOp" input: "bert/embeddings/word_embeddings" input: "Identity_2" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_3/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_3/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_3/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_3" op: "RestoreV2" input: "checkpoint_initializer_3/prefix" input: "checkpoint_initializer_3/tensor_names" input: "checkpoint_initializer_3/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_3" op: "Identity" input: "checkpoint_initializer_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_3" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias" input: "Identity_3" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_4/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_4/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_4/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_4" op: "RestoreV2" input: "checkpoint_initializer_4/prefix" input: "checkpoint_initializer_4/tensor_names" input: "checkpoint_initializer_4/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_4" op: "Identity" input: "checkpoint_initializer_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_4" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel" input: "Identity_4" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_5/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_5/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_5/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_5" op: "RestoreV2" input: "checkpoint_initializer_5/prefix" input: "checkpoint_initializer_5/tensor_names" input: "checkpoint_initializer_5/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_5" op: "Identity" input: "checkpoint_initializer_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_5" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias" input: "Identity_5" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_6/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_6/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_6/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_6" op: "RestoreV2" input: "checkpoint_initializer_6/prefix" input: "checkpoint_initializer_6/tensor_names" input: "checkpoint_initializer_6/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_6" op: "Identity" input: "checkpoint_initializer_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_6" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel" input: "Identity_6" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_7/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_7/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_7/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_7" op: "RestoreV2" input: "checkpoint_initializer_7/prefix" input: "checkpoint_initializer_7/tensor_names" input: "checkpoint_initializer_7/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_7" op: "Identity" input: "checkpoint_initializer_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_7" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias" input: "Identity_7" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_8/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_8/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_8/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_8" op: "RestoreV2" input: "checkpoint_initializer_8/prefix" input: "checkpoint_initializer_8/tensor_names" input: "checkpoint_initializer_8/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_8" op: "Identity" input: "checkpoint_initializer_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_8" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel" input: "Identity_8" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_9/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_9/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_9/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_9" op: "RestoreV2" input: "checkpoint_initializer_9/prefix" input: "checkpoint_initializer_9/tensor_names" input: "checkpoint_initializer_9/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_9" op: "Identity" input: "checkpoint_initializer_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_9" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias" input: "Identity_9" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_10/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_10/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_10/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_10" op: "RestoreV2" input: "checkpoint_initializer_10/prefix" input: "checkpoint_initializer_10/tensor_names" input: "checkpoint_initializer_10/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_10" op: "Identity" input: "checkpoint_initializer_10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_10" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel" input: "Identity_10" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_11/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_11/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_11/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_11" op: "RestoreV2" input: "checkpoint_initializer_11/prefix" input: "checkpoint_initializer_11/tensor_names" input: "checkpoint_initializer_11/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_11" op: "Identity" input: "checkpoint_initializer_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_11" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias" input: "Identity_11" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_12/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_12/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_12/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_12" op: "RestoreV2" input: "checkpoint_initializer_12/prefix" input: "checkpoint_initializer_12/tensor_names" input: "checkpoint_initializer_12/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_12" op: "Identity" input: "checkpoint_initializer_12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_12" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel" input: "Identity_12" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_13/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_13/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/output/dense/bias" } } } } node { name: "checkpoint_initializer_13/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_13" op: "RestoreV2" input: "checkpoint_initializer_13/prefix" input: "checkpoint_initializer_13/tensor_names" input: "checkpoint_initializer_13/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_13" op: "Identity" input: "checkpoint_initializer_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_13" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/bias" input: "Identity_13" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_14/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_14/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_0/output/dense/kernel" } } } } node { name: "checkpoint_initializer_14/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_14" op: "RestoreV2" input: "checkpoint_initializer_14/prefix" input: "checkpoint_initializer_14/tensor_names" input: "checkpoint_initializer_14/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_14" op: "Identity" input: "checkpoint_initializer_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_14" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/kernel" input: "Identity_14" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_15/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_15/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_15/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_15" op: "RestoreV2" input: "checkpoint_initializer_15/prefix" input: "checkpoint_initializer_15/tensor_names" input: "checkpoint_initializer_15/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_15" op: "Identity" input: "checkpoint_initializer_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_15" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias" input: "Identity_15" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_16/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_16/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_16/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_16" op: "RestoreV2" input: "checkpoint_initializer_16/prefix" input: "checkpoint_initializer_16/tensor_names" input: "checkpoint_initializer_16/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_16" op: "Identity" input: "checkpoint_initializer_16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_16" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel" input: "Identity_16" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_17/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_17/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_17/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_17" op: "RestoreV2" input: "checkpoint_initializer_17/prefix" input: "checkpoint_initializer_17/tensor_names" input: "checkpoint_initializer_17/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_17" op: "Identity" input: "checkpoint_initializer_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_17" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias" input: "Identity_17" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_18/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_18/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_18/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_18" op: "RestoreV2" input: "checkpoint_initializer_18/prefix" input: "checkpoint_initializer_18/tensor_names" input: "checkpoint_initializer_18/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_18" op: "Identity" input: "checkpoint_initializer_18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_18" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel" input: "Identity_18" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_19/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_19/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_19/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_19" op: "RestoreV2" input: "checkpoint_initializer_19/prefix" input: "checkpoint_initializer_19/tensor_names" input: "checkpoint_initializer_19/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_19" op: "Identity" input: "checkpoint_initializer_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_19" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias" input: "Identity_19" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_20/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_20/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_20/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_20" op: "RestoreV2" input: "checkpoint_initializer_20/prefix" input: "checkpoint_initializer_20/tensor_names" input: "checkpoint_initializer_20/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_20" op: "Identity" input: "checkpoint_initializer_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_20" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel" input: "Identity_20" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_21/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_21/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_21/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_21" op: "RestoreV2" input: "checkpoint_initializer_21/prefix" input: "checkpoint_initializer_21/tensor_names" input: "checkpoint_initializer_21/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_21" op: "Identity" input: "checkpoint_initializer_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_21" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias" input: "Identity_21" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_22/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_22/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_22/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_22" op: "RestoreV2" input: "checkpoint_initializer_22/prefix" input: "checkpoint_initializer_22/tensor_names" input: "checkpoint_initializer_22/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_22" op: "Identity" input: "checkpoint_initializer_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_22" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel" input: "Identity_22" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_23/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_23/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_23/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_23" op: "RestoreV2" input: "checkpoint_initializer_23/prefix" input: "checkpoint_initializer_23/tensor_names" input: "checkpoint_initializer_23/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_23" op: "Identity" input: "checkpoint_initializer_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_23" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias" input: "Identity_23" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_24/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_24/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_24/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_24" op: "RestoreV2" input: "checkpoint_initializer_24/prefix" input: "checkpoint_initializer_24/tensor_names" input: "checkpoint_initializer_24/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_24" op: "Identity" input: "checkpoint_initializer_24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_24" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel" input: "Identity_24" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_25/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_25/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/output/dense/bias" } } } } node { name: "checkpoint_initializer_25/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_25" op: "RestoreV2" input: "checkpoint_initializer_25/prefix" input: "checkpoint_initializer_25/tensor_names" input: "checkpoint_initializer_25/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_25" op: "Identity" input: "checkpoint_initializer_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_25" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/bias" input: "Identity_25" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_26/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_26/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_1/output/dense/kernel" } } } } node { name: "checkpoint_initializer_26/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_26" op: "RestoreV2" input: "checkpoint_initializer_26/prefix" input: "checkpoint_initializer_26/tensor_names" input: "checkpoint_initializer_26/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_26" op: "Identity" input: "checkpoint_initializer_26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_26" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/kernel" input: "Identity_26" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_27/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_27/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_27/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_27" op: "RestoreV2" input: "checkpoint_initializer_27/prefix" input: "checkpoint_initializer_27/tensor_names" input: "checkpoint_initializer_27/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_27" op: "Identity" input: "checkpoint_initializer_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_27" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias" input: "Identity_27" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_28/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_28/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_28/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_28" op: "RestoreV2" input: "checkpoint_initializer_28/prefix" input: "checkpoint_initializer_28/tensor_names" input: "checkpoint_initializer_28/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_28" op: "Identity" input: "checkpoint_initializer_28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_28" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel" input: "Identity_28" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_29/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_29/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_29/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_29" op: "RestoreV2" input: "checkpoint_initializer_29/prefix" input: "checkpoint_initializer_29/tensor_names" input: "checkpoint_initializer_29/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_29" op: "Identity" input: "checkpoint_initializer_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_29" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias" input: "Identity_29" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_30/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_30/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_30/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_30" op: "RestoreV2" input: "checkpoint_initializer_30/prefix" input: "checkpoint_initializer_30/tensor_names" input: "checkpoint_initializer_30/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_30" op: "Identity" input: "checkpoint_initializer_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_30" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel" input: "Identity_30" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_31/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_31/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_31/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_31" op: "RestoreV2" input: "checkpoint_initializer_31/prefix" input: "checkpoint_initializer_31/tensor_names" input: "checkpoint_initializer_31/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_31" op: "Identity" input: "checkpoint_initializer_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_31" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias" input: "Identity_31" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_32/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_32/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_32/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_32" op: "RestoreV2" input: "checkpoint_initializer_32/prefix" input: "checkpoint_initializer_32/tensor_names" input: "checkpoint_initializer_32/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_32" op: "Identity" input: "checkpoint_initializer_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_32" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel" input: "Identity_32" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_33/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_33/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_33/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_33" op: "RestoreV2" input: "checkpoint_initializer_33/prefix" input: "checkpoint_initializer_33/tensor_names" input: "checkpoint_initializer_33/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_33" op: "Identity" input: "checkpoint_initializer_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_33" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias" input: "Identity_33" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_34/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_34/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_34/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_34" op: "RestoreV2" input: "checkpoint_initializer_34/prefix" input: "checkpoint_initializer_34/tensor_names" input: "checkpoint_initializer_34/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_34" op: "Identity" input: "checkpoint_initializer_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_34" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel" input: "Identity_34" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_35/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_35/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_35/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_35" op: "RestoreV2" input: "checkpoint_initializer_35/prefix" input: "checkpoint_initializer_35/tensor_names" input: "checkpoint_initializer_35/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_35" op: "Identity" input: "checkpoint_initializer_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_35" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias" input: "Identity_35" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_36/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_36/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_36/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_36" op: "RestoreV2" input: "checkpoint_initializer_36/prefix" input: "checkpoint_initializer_36/tensor_names" input: "checkpoint_initializer_36/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_36" op: "Identity" input: "checkpoint_initializer_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_36" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel" input: "Identity_36" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_37/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_37/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/output/dense/bias" } } } } node { name: "checkpoint_initializer_37/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_37" op: "RestoreV2" input: "checkpoint_initializer_37/prefix" input: "checkpoint_initializer_37/tensor_names" input: "checkpoint_initializer_37/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_37" op: "Identity" input: "checkpoint_initializer_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_37" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/bias" input: "Identity_37" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_38/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_38/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_10/output/dense/kernel" } } } } node { name: "checkpoint_initializer_38/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_38" op: "RestoreV2" input: "checkpoint_initializer_38/prefix" input: "checkpoint_initializer_38/tensor_names" input: "checkpoint_initializer_38/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_38" op: "Identity" input: "checkpoint_initializer_38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_38" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/kernel" input: "Identity_38" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_39/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_39/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_39/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_39" op: "RestoreV2" input: "checkpoint_initializer_39/prefix" input: "checkpoint_initializer_39/tensor_names" input: "checkpoint_initializer_39/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_39" op: "Identity" input: "checkpoint_initializer_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_39" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias" input: "Identity_39" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_40/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_40/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_40/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_40" op: "RestoreV2" input: "checkpoint_initializer_40/prefix" input: "checkpoint_initializer_40/tensor_names" input: "checkpoint_initializer_40/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_40" op: "Identity" input: "checkpoint_initializer_40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_40" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel" input: "Identity_40" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_41/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_41/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_41/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_41" op: "RestoreV2" input: "checkpoint_initializer_41/prefix" input: "checkpoint_initializer_41/tensor_names" input: "checkpoint_initializer_41/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_41" op: "Identity" input: "checkpoint_initializer_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_41" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias" input: "Identity_41" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_42/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_42/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_42/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_42" op: "RestoreV2" input: "checkpoint_initializer_42/prefix" input: "checkpoint_initializer_42/tensor_names" input: "checkpoint_initializer_42/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_42" op: "Identity" input: "checkpoint_initializer_42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_42" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel" input: "Identity_42" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_43/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_43/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_43/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_43" op: "RestoreV2" input: "checkpoint_initializer_43/prefix" input: "checkpoint_initializer_43/tensor_names" input: "checkpoint_initializer_43/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_43" op: "Identity" input: "checkpoint_initializer_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_43" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias" input: "Identity_43" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_44/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_44/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_44/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_44" op: "RestoreV2" input: "checkpoint_initializer_44/prefix" input: "checkpoint_initializer_44/tensor_names" input: "checkpoint_initializer_44/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_44" op: "Identity" input: "checkpoint_initializer_44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_44" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel" input: "Identity_44" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_45/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_45/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_45/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_45" op: "RestoreV2" input: "checkpoint_initializer_45/prefix" input: "checkpoint_initializer_45/tensor_names" input: "checkpoint_initializer_45/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_45" op: "Identity" input: "checkpoint_initializer_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_45" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias" input: "Identity_45" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_46/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_46/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_46/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_46" op: "RestoreV2" input: "checkpoint_initializer_46/prefix" input: "checkpoint_initializer_46/tensor_names" input: "checkpoint_initializer_46/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_46" op: "Identity" input: "checkpoint_initializer_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_46" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel" input: "Identity_46" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_47/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_47/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_47/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_47" op: "RestoreV2" input: "checkpoint_initializer_47/prefix" input: "checkpoint_initializer_47/tensor_names" input: "checkpoint_initializer_47/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_47" op: "Identity" input: "checkpoint_initializer_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_47" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias" input: "Identity_47" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_48/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_48/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_48/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_48" op: "RestoreV2" input: "checkpoint_initializer_48/prefix" input: "checkpoint_initializer_48/tensor_names" input: "checkpoint_initializer_48/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_48" op: "Identity" input: "checkpoint_initializer_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_48" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel" input: "Identity_48" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_49/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_49/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/output/dense/bias" } } } } node { name: "checkpoint_initializer_49/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_49" op: "RestoreV2" input: "checkpoint_initializer_49/prefix" input: "checkpoint_initializer_49/tensor_names" input: "checkpoint_initializer_49/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_49" op: "Identity" input: "checkpoint_initializer_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_49" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/bias" input: "Identity_49" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_50/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_50/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_11/output/dense/kernel" } } } } node { name: "checkpoint_initializer_50/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_50" op: "RestoreV2" input: "checkpoint_initializer_50/prefix" input: "checkpoint_initializer_50/tensor_names" input: "checkpoint_initializer_50/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_50" op: "Identity" input: "checkpoint_initializer_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_50" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/kernel" input: "Identity_50" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_51/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_51/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_51/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_51" op: "RestoreV2" input: "checkpoint_initializer_51/prefix" input: "checkpoint_initializer_51/tensor_names" input: "checkpoint_initializer_51/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_51" op: "Identity" input: "checkpoint_initializer_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_51" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias" input: "Identity_51" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_52/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_52/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_52/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_52" op: "RestoreV2" input: "checkpoint_initializer_52/prefix" input: "checkpoint_initializer_52/tensor_names" input: "checkpoint_initializer_52/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_52" op: "Identity" input: "checkpoint_initializer_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_52" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel" input: "Identity_52" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_53/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_53/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_53/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_53" op: "RestoreV2" input: "checkpoint_initializer_53/prefix" input: "checkpoint_initializer_53/tensor_names" input: "checkpoint_initializer_53/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_53" op: "Identity" input: "checkpoint_initializer_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_53" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias" input: "Identity_53" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_54/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_54/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_54/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_54" op: "RestoreV2" input: "checkpoint_initializer_54/prefix" input: "checkpoint_initializer_54/tensor_names" input: "checkpoint_initializer_54/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_54" op: "Identity" input: "checkpoint_initializer_54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_54" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel" input: "Identity_54" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_55/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_55/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_55/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_55" op: "RestoreV2" input: "checkpoint_initializer_55/prefix" input: "checkpoint_initializer_55/tensor_names" input: "checkpoint_initializer_55/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_55" op: "Identity" input: "checkpoint_initializer_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_55" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias" input: "Identity_55" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_56/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_56/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_56/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_56" op: "RestoreV2" input: "checkpoint_initializer_56/prefix" input: "checkpoint_initializer_56/tensor_names" input: "checkpoint_initializer_56/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_56" op: "Identity" input: "checkpoint_initializer_56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_56" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel" input: "Identity_56" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_57/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_57/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_57/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_57" op: "RestoreV2" input: "checkpoint_initializer_57/prefix" input: "checkpoint_initializer_57/tensor_names" input: "checkpoint_initializer_57/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_57" op: "Identity" input: "checkpoint_initializer_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_57" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias" input: "Identity_57" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_58/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_58/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_58/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_58" op: "RestoreV2" input: "checkpoint_initializer_58/prefix" input: "checkpoint_initializer_58/tensor_names" input: "checkpoint_initializer_58/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_58" op: "Identity" input: "checkpoint_initializer_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_58" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel" input: "Identity_58" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_59/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_59/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_59/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_59" op: "RestoreV2" input: "checkpoint_initializer_59/prefix" input: "checkpoint_initializer_59/tensor_names" input: "checkpoint_initializer_59/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_59" op: "Identity" input: "checkpoint_initializer_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_59" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias" input: "Identity_59" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_60/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_60/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_60/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_60" op: "RestoreV2" input: "checkpoint_initializer_60/prefix" input: "checkpoint_initializer_60/tensor_names" input: "checkpoint_initializer_60/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_60" op: "Identity" input: "checkpoint_initializer_60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_60" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel" input: "Identity_60" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_61/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_61/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/output/dense/bias" } } } } node { name: "checkpoint_initializer_61/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_61" op: "RestoreV2" input: "checkpoint_initializer_61/prefix" input: "checkpoint_initializer_61/tensor_names" input: "checkpoint_initializer_61/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_61" op: "Identity" input: "checkpoint_initializer_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_61" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/bias" input: "Identity_61" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_62/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_62/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_2/output/dense/kernel" } } } } node { name: "checkpoint_initializer_62/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_62" op: "RestoreV2" input: "checkpoint_initializer_62/prefix" input: "checkpoint_initializer_62/tensor_names" input: "checkpoint_initializer_62/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_62" op: "Identity" input: "checkpoint_initializer_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_62" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/kernel" input: "Identity_62" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_63/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_63/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_63/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_63" op: "RestoreV2" input: "checkpoint_initializer_63/prefix" input: "checkpoint_initializer_63/tensor_names" input: "checkpoint_initializer_63/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_63" op: "Identity" input: "checkpoint_initializer_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_63" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias" input: "Identity_63" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_64/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_64/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_64/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_64" op: "RestoreV2" input: "checkpoint_initializer_64/prefix" input: "checkpoint_initializer_64/tensor_names" input: "checkpoint_initializer_64/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_64" op: "Identity" input: "checkpoint_initializer_64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_64" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel" input: "Identity_64" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_65/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_65/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_65/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_65" op: "RestoreV2" input: "checkpoint_initializer_65/prefix" input: "checkpoint_initializer_65/tensor_names" input: "checkpoint_initializer_65/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_65" op: "Identity" input: "checkpoint_initializer_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_65" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias" input: "Identity_65" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_66/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_66/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_66/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_66" op: "RestoreV2" input: "checkpoint_initializer_66/prefix" input: "checkpoint_initializer_66/tensor_names" input: "checkpoint_initializer_66/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_66" op: "Identity" input: "checkpoint_initializer_66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_66" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel" input: "Identity_66" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_67/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_67/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_67/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_67" op: "RestoreV2" input: "checkpoint_initializer_67/prefix" input: "checkpoint_initializer_67/tensor_names" input: "checkpoint_initializer_67/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_67" op: "Identity" input: "checkpoint_initializer_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_67" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias" input: "Identity_67" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_68/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_68/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_68/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_68" op: "RestoreV2" input: "checkpoint_initializer_68/prefix" input: "checkpoint_initializer_68/tensor_names" input: "checkpoint_initializer_68/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_68" op: "Identity" input: "checkpoint_initializer_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_68" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel" input: "Identity_68" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_69/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_69/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_69/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_69" op: "RestoreV2" input: "checkpoint_initializer_69/prefix" input: "checkpoint_initializer_69/tensor_names" input: "checkpoint_initializer_69/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_69" op: "Identity" input: "checkpoint_initializer_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_69" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias" input: "Identity_69" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_70/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_70/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_70/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_70" op: "RestoreV2" input: "checkpoint_initializer_70/prefix" input: "checkpoint_initializer_70/tensor_names" input: "checkpoint_initializer_70/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_70" op: "Identity" input: "checkpoint_initializer_70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_70" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel" input: "Identity_70" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_71/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_71/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_71/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_71" op: "RestoreV2" input: "checkpoint_initializer_71/prefix" input: "checkpoint_initializer_71/tensor_names" input: "checkpoint_initializer_71/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_71" op: "Identity" input: "checkpoint_initializer_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_71" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias" input: "Identity_71" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_72/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_72/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_72/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_72" op: "RestoreV2" input: "checkpoint_initializer_72/prefix" input: "checkpoint_initializer_72/tensor_names" input: "checkpoint_initializer_72/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_72" op: "Identity" input: "checkpoint_initializer_72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_72" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel" input: "Identity_72" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_73/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_73/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/output/dense/bias" } } } } node { name: "checkpoint_initializer_73/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_73" op: "RestoreV2" input: "checkpoint_initializer_73/prefix" input: "checkpoint_initializer_73/tensor_names" input: "checkpoint_initializer_73/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_73" op: "Identity" input: "checkpoint_initializer_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_73" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/bias" input: "Identity_73" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_74/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_74/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_3/output/dense/kernel" } } } } node { name: "checkpoint_initializer_74/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_74" op: "RestoreV2" input: "checkpoint_initializer_74/prefix" input: "checkpoint_initializer_74/tensor_names" input: "checkpoint_initializer_74/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_74" op: "Identity" input: "checkpoint_initializer_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_74" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/kernel" input: "Identity_74" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_75/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_75/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_75/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_75" op: "RestoreV2" input: "checkpoint_initializer_75/prefix" input: "checkpoint_initializer_75/tensor_names" input: "checkpoint_initializer_75/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_75" op: "Identity" input: "checkpoint_initializer_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_75" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias" input: "Identity_75" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_76/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_76/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_76/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_76" op: "RestoreV2" input: "checkpoint_initializer_76/prefix" input: "checkpoint_initializer_76/tensor_names" input: "checkpoint_initializer_76/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_76" op: "Identity" input: "checkpoint_initializer_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_76" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel" input: "Identity_76" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_77/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_77/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_77/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_77" op: "RestoreV2" input: "checkpoint_initializer_77/prefix" input: "checkpoint_initializer_77/tensor_names" input: "checkpoint_initializer_77/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_77" op: "Identity" input: "checkpoint_initializer_77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_77" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias" input: "Identity_77" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_78/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_78/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_78/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_78" op: "RestoreV2" input: "checkpoint_initializer_78/prefix" input: "checkpoint_initializer_78/tensor_names" input: "checkpoint_initializer_78/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_78" op: "Identity" input: "checkpoint_initializer_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_78" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel" input: "Identity_78" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_79/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_79/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_79/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_79" op: "RestoreV2" input: "checkpoint_initializer_79/prefix" input: "checkpoint_initializer_79/tensor_names" input: "checkpoint_initializer_79/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_79" op: "Identity" input: "checkpoint_initializer_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_79" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias" input: "Identity_79" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_80/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_80/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_80/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_80" op: "RestoreV2" input: "checkpoint_initializer_80/prefix" input: "checkpoint_initializer_80/tensor_names" input: "checkpoint_initializer_80/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_80" op: "Identity" input: "checkpoint_initializer_80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_80" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel" input: "Identity_80" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_81/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_81/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_81/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_81" op: "RestoreV2" input: "checkpoint_initializer_81/prefix" input: "checkpoint_initializer_81/tensor_names" input: "checkpoint_initializer_81/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_81" op: "Identity" input: "checkpoint_initializer_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_81" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias" input: "Identity_81" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_82/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_82/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_82/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_82" op: "RestoreV2" input: "checkpoint_initializer_82/prefix" input: "checkpoint_initializer_82/tensor_names" input: "checkpoint_initializer_82/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_82" op: "Identity" input: "checkpoint_initializer_82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_82" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel" input: "Identity_82" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_83/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_83/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_83/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_83" op: "RestoreV2" input: "checkpoint_initializer_83/prefix" input: "checkpoint_initializer_83/tensor_names" input: "checkpoint_initializer_83/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_83" op: "Identity" input: "checkpoint_initializer_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_83" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias" input: "Identity_83" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_84/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_84/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_84/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_84" op: "RestoreV2" input: "checkpoint_initializer_84/prefix" input: "checkpoint_initializer_84/tensor_names" input: "checkpoint_initializer_84/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_84" op: "Identity" input: "checkpoint_initializer_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_84" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel" input: "Identity_84" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_85/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_85/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/output/dense/bias" } } } } node { name: "checkpoint_initializer_85/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_85" op: "RestoreV2" input: "checkpoint_initializer_85/prefix" input: "checkpoint_initializer_85/tensor_names" input: "checkpoint_initializer_85/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_85" op: "Identity" input: "checkpoint_initializer_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_85" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/bias" input: "Identity_85" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_86/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_86/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_4/output/dense/kernel" } } } } node { name: "checkpoint_initializer_86/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_86" op: "RestoreV2" input: "checkpoint_initializer_86/prefix" input: "checkpoint_initializer_86/tensor_names" input: "checkpoint_initializer_86/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_86" op: "Identity" input: "checkpoint_initializer_86" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_86" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/kernel" input: "Identity_86" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_87/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_87/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_87/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_87" op: "RestoreV2" input: "checkpoint_initializer_87/prefix" input: "checkpoint_initializer_87/tensor_names" input: "checkpoint_initializer_87/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_87" op: "Identity" input: "checkpoint_initializer_87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_87" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias" input: "Identity_87" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_88/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_88/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_88/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_88" op: "RestoreV2" input: "checkpoint_initializer_88/prefix" input: "checkpoint_initializer_88/tensor_names" input: "checkpoint_initializer_88/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_88" op: "Identity" input: "checkpoint_initializer_88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_88" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel" input: "Identity_88" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_89/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_89/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_89/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_89" op: "RestoreV2" input: "checkpoint_initializer_89/prefix" input: "checkpoint_initializer_89/tensor_names" input: "checkpoint_initializer_89/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_89" op: "Identity" input: "checkpoint_initializer_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_89" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias" input: "Identity_89" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_90/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_90/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_90/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_90" op: "RestoreV2" input: "checkpoint_initializer_90/prefix" input: "checkpoint_initializer_90/tensor_names" input: "checkpoint_initializer_90/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_90" op: "Identity" input: "checkpoint_initializer_90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_90" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel" input: "Identity_90" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_91/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_91/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_91/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_91" op: "RestoreV2" input: "checkpoint_initializer_91/prefix" input: "checkpoint_initializer_91/tensor_names" input: "checkpoint_initializer_91/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_91" op: "Identity" input: "checkpoint_initializer_91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_91" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias" input: "Identity_91" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_92/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_92/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_92/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_92" op: "RestoreV2" input: "checkpoint_initializer_92/prefix" input: "checkpoint_initializer_92/tensor_names" input: "checkpoint_initializer_92/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_92" op: "Identity" input: "checkpoint_initializer_92" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_92" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel" input: "Identity_92" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_93/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_93/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_93/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_93" op: "RestoreV2" input: "checkpoint_initializer_93/prefix" input: "checkpoint_initializer_93/tensor_names" input: "checkpoint_initializer_93/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_93" op: "Identity" input: "checkpoint_initializer_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_93" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias" input: "Identity_93" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_94/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_94/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_94/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_94" op: "RestoreV2" input: "checkpoint_initializer_94/prefix" input: "checkpoint_initializer_94/tensor_names" input: "checkpoint_initializer_94/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_94" op: "Identity" input: "checkpoint_initializer_94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_94" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel" input: "Identity_94" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_95/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_95/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_95/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_95" op: "RestoreV2" input: "checkpoint_initializer_95/prefix" input: "checkpoint_initializer_95/tensor_names" input: "checkpoint_initializer_95/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_95" op: "Identity" input: "checkpoint_initializer_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_95" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias" input: "Identity_95" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_96/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_96/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_96/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_96" op: "RestoreV2" input: "checkpoint_initializer_96/prefix" input: "checkpoint_initializer_96/tensor_names" input: "checkpoint_initializer_96/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_96" op: "Identity" input: "checkpoint_initializer_96" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_96" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel" input: "Identity_96" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_97/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_97/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/output/dense/bias" } } } } node { name: "checkpoint_initializer_97/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_97" op: "RestoreV2" input: "checkpoint_initializer_97/prefix" input: "checkpoint_initializer_97/tensor_names" input: "checkpoint_initializer_97/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_97" op: "Identity" input: "checkpoint_initializer_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_97" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/bias" input: "Identity_97" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_98/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_98/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_5/output/dense/kernel" } } } } node { name: "checkpoint_initializer_98/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_98" op: "RestoreV2" input: "checkpoint_initializer_98/prefix" input: "checkpoint_initializer_98/tensor_names" input: "checkpoint_initializer_98/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_98" op: "Identity" input: "checkpoint_initializer_98" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_98" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/kernel" input: "Identity_98" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_99/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_99/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_99/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_99" op: "RestoreV2" input: "checkpoint_initializer_99/prefix" input: "checkpoint_initializer_99/tensor_names" input: "checkpoint_initializer_99/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_99" op: "Identity" input: "checkpoint_initializer_99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_99" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias" input: "Identity_99" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_100/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_100/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_100/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_100" op: "RestoreV2" input: "checkpoint_initializer_100/prefix" input: "checkpoint_initializer_100/tensor_names" input: "checkpoint_initializer_100/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_100" op: "Identity" input: "checkpoint_initializer_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_100" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel" input: "Identity_100" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_101/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_101/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_101/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_101" op: "RestoreV2" input: "checkpoint_initializer_101/prefix" input: "checkpoint_initializer_101/tensor_names" input: "checkpoint_initializer_101/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_101" op: "Identity" input: "checkpoint_initializer_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_101" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias" input: "Identity_101" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_102/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_102/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_102/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_102" op: "RestoreV2" input: "checkpoint_initializer_102/prefix" input: "checkpoint_initializer_102/tensor_names" input: "checkpoint_initializer_102/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_102" op: "Identity" input: "checkpoint_initializer_102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_102" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel" input: "Identity_102" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_103/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_103/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_103/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_103" op: "RestoreV2" input: "checkpoint_initializer_103/prefix" input: "checkpoint_initializer_103/tensor_names" input: "checkpoint_initializer_103/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_103" op: "Identity" input: "checkpoint_initializer_103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_103" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias" input: "Identity_103" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_104/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_104/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_104/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_104" op: "RestoreV2" input: "checkpoint_initializer_104/prefix" input: "checkpoint_initializer_104/tensor_names" input: "checkpoint_initializer_104/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_104" op: "Identity" input: "checkpoint_initializer_104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_104" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel" input: "Identity_104" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_105/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_105/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_105/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_105" op: "RestoreV2" input: "checkpoint_initializer_105/prefix" input: "checkpoint_initializer_105/tensor_names" input: "checkpoint_initializer_105/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_105" op: "Identity" input: "checkpoint_initializer_105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_105" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias" input: "Identity_105" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_106/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_106/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_106/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_106" op: "RestoreV2" input: "checkpoint_initializer_106/prefix" input: "checkpoint_initializer_106/tensor_names" input: "checkpoint_initializer_106/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_106" op: "Identity" input: "checkpoint_initializer_106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_106" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel" input: "Identity_106" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_107/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_107/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_107/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_107" op: "RestoreV2" input: "checkpoint_initializer_107/prefix" input: "checkpoint_initializer_107/tensor_names" input: "checkpoint_initializer_107/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_107" op: "Identity" input: "checkpoint_initializer_107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_107" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias" input: "Identity_107" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_108/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_108/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_108/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_108" op: "RestoreV2" input: "checkpoint_initializer_108/prefix" input: "checkpoint_initializer_108/tensor_names" input: "checkpoint_initializer_108/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_108" op: "Identity" input: "checkpoint_initializer_108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_108" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel" input: "Identity_108" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_109/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_109/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/output/dense/bias" } } } } node { name: "checkpoint_initializer_109/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_109" op: "RestoreV2" input: "checkpoint_initializer_109/prefix" input: "checkpoint_initializer_109/tensor_names" input: "checkpoint_initializer_109/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_109" op: "Identity" input: "checkpoint_initializer_109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_109" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/bias" input: "Identity_109" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_110/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_110/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_6/output/dense/kernel" } } } } node { name: "checkpoint_initializer_110/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_110" op: "RestoreV2" input: "checkpoint_initializer_110/prefix" input: "checkpoint_initializer_110/tensor_names" input: "checkpoint_initializer_110/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_110" op: "Identity" input: "checkpoint_initializer_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_110" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/kernel" input: "Identity_110" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_111/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_111/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_111/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_111" op: "RestoreV2" input: "checkpoint_initializer_111/prefix" input: "checkpoint_initializer_111/tensor_names" input: "checkpoint_initializer_111/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_111" op: "Identity" input: "checkpoint_initializer_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_111" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias" input: "Identity_111" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_112/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_112/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_112/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_112" op: "RestoreV2" input: "checkpoint_initializer_112/prefix" input: "checkpoint_initializer_112/tensor_names" input: "checkpoint_initializer_112/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_112" op: "Identity" input: "checkpoint_initializer_112" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_112" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel" input: "Identity_112" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_113/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_113/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_113/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_113" op: "RestoreV2" input: "checkpoint_initializer_113/prefix" input: "checkpoint_initializer_113/tensor_names" input: "checkpoint_initializer_113/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_113" op: "Identity" input: "checkpoint_initializer_113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_113" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias" input: "Identity_113" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_114/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_114/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_114/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_114" op: "RestoreV2" input: "checkpoint_initializer_114/prefix" input: "checkpoint_initializer_114/tensor_names" input: "checkpoint_initializer_114/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_114" op: "Identity" input: "checkpoint_initializer_114" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_114" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel" input: "Identity_114" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_115/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_115/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_115/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_115" op: "RestoreV2" input: "checkpoint_initializer_115/prefix" input: "checkpoint_initializer_115/tensor_names" input: "checkpoint_initializer_115/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_115" op: "Identity" input: "checkpoint_initializer_115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_115" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias" input: "Identity_115" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_116/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_116/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_116/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_116" op: "RestoreV2" input: "checkpoint_initializer_116/prefix" input: "checkpoint_initializer_116/tensor_names" input: "checkpoint_initializer_116/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_116" op: "Identity" input: "checkpoint_initializer_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_116" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel" input: "Identity_116" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_117/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_117/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_117/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_117" op: "RestoreV2" input: "checkpoint_initializer_117/prefix" input: "checkpoint_initializer_117/tensor_names" input: "checkpoint_initializer_117/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_117" op: "Identity" input: "checkpoint_initializer_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_117" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias" input: "Identity_117" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_118/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_118/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_118/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_118" op: "RestoreV2" input: "checkpoint_initializer_118/prefix" input: "checkpoint_initializer_118/tensor_names" input: "checkpoint_initializer_118/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_118" op: "Identity" input: "checkpoint_initializer_118" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_118" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel" input: "Identity_118" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_119/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_119/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_119/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_119" op: "RestoreV2" input: "checkpoint_initializer_119/prefix" input: "checkpoint_initializer_119/tensor_names" input: "checkpoint_initializer_119/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_119" op: "Identity" input: "checkpoint_initializer_119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_119" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias" input: "Identity_119" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_120/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_120/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_120/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_120" op: "RestoreV2" input: "checkpoint_initializer_120/prefix" input: "checkpoint_initializer_120/tensor_names" input: "checkpoint_initializer_120/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_120" op: "Identity" input: "checkpoint_initializer_120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_120" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel" input: "Identity_120" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_121/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_121/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/output/dense/bias" } } } } node { name: "checkpoint_initializer_121/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_121" op: "RestoreV2" input: "checkpoint_initializer_121/prefix" input: "checkpoint_initializer_121/tensor_names" input: "checkpoint_initializer_121/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_121" op: "Identity" input: "checkpoint_initializer_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_121" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/bias" input: "Identity_121" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_122/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_122/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_7/output/dense/kernel" } } } } node { name: "checkpoint_initializer_122/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_122" op: "RestoreV2" input: "checkpoint_initializer_122/prefix" input: "checkpoint_initializer_122/tensor_names" input: "checkpoint_initializer_122/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_122" op: "Identity" input: "checkpoint_initializer_122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_122" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/kernel" input: "Identity_122" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_123/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_123/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_123/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_123" op: "RestoreV2" input: "checkpoint_initializer_123/prefix" input: "checkpoint_initializer_123/tensor_names" input: "checkpoint_initializer_123/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_123" op: "Identity" input: "checkpoint_initializer_123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_123" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias" input: "Identity_123" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_124/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_124/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_124/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_124" op: "RestoreV2" input: "checkpoint_initializer_124/prefix" input: "checkpoint_initializer_124/tensor_names" input: "checkpoint_initializer_124/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_124" op: "Identity" input: "checkpoint_initializer_124" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_124" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel" input: "Identity_124" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_125/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_125/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_125/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_125" op: "RestoreV2" input: "checkpoint_initializer_125/prefix" input: "checkpoint_initializer_125/tensor_names" input: "checkpoint_initializer_125/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_125" op: "Identity" input: "checkpoint_initializer_125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_125" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias" input: "Identity_125" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_126/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_126/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_126/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_126" op: "RestoreV2" input: "checkpoint_initializer_126/prefix" input: "checkpoint_initializer_126/tensor_names" input: "checkpoint_initializer_126/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_126" op: "Identity" input: "checkpoint_initializer_126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_126" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel" input: "Identity_126" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_127/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_127/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_127/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_127" op: "RestoreV2" input: "checkpoint_initializer_127/prefix" input: "checkpoint_initializer_127/tensor_names" input: "checkpoint_initializer_127/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_127" op: "Identity" input: "checkpoint_initializer_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_127" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias" input: "Identity_127" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_128/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_128/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_128/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_128" op: "RestoreV2" input: "checkpoint_initializer_128/prefix" input: "checkpoint_initializer_128/tensor_names" input: "checkpoint_initializer_128/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_128" op: "Identity" input: "checkpoint_initializer_128" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_128" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel" input: "Identity_128" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_129/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_129/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_129/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_129" op: "RestoreV2" input: "checkpoint_initializer_129/prefix" input: "checkpoint_initializer_129/tensor_names" input: "checkpoint_initializer_129/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_129" op: "Identity" input: "checkpoint_initializer_129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_129" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias" input: "Identity_129" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_130/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_130/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_130/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_130" op: "RestoreV2" input: "checkpoint_initializer_130/prefix" input: "checkpoint_initializer_130/tensor_names" input: "checkpoint_initializer_130/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_130" op: "Identity" input: "checkpoint_initializer_130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_130" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel" input: "Identity_130" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_131/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_131/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_131/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_131" op: "RestoreV2" input: "checkpoint_initializer_131/prefix" input: "checkpoint_initializer_131/tensor_names" input: "checkpoint_initializer_131/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_131" op: "Identity" input: "checkpoint_initializer_131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_131" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias" input: "Identity_131" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_132/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_132/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_132/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_132" op: "RestoreV2" input: "checkpoint_initializer_132/prefix" input: "checkpoint_initializer_132/tensor_names" input: "checkpoint_initializer_132/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_132" op: "Identity" input: "checkpoint_initializer_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_132" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel" input: "Identity_132" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_133/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_133/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/output/dense/bias" } } } } node { name: "checkpoint_initializer_133/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_133" op: "RestoreV2" input: "checkpoint_initializer_133/prefix" input: "checkpoint_initializer_133/tensor_names" input: "checkpoint_initializer_133/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_133" op: "Identity" input: "checkpoint_initializer_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_133" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/bias" input: "Identity_133" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_134/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_134/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_8/output/dense/kernel" } } } } node { name: "checkpoint_initializer_134/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_134" op: "RestoreV2" input: "checkpoint_initializer_134/prefix" input: "checkpoint_initializer_134/tensor_names" input: "checkpoint_initializer_134/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_134" op: "Identity" input: "checkpoint_initializer_134" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_134" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/kernel" input: "Identity_134" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_135/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_135/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/output/dense/bias" } } } } node { name: "checkpoint_initializer_135/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_135" op: "RestoreV2" input: "checkpoint_initializer_135/prefix" input: "checkpoint_initializer_135/tensor_names" input: "checkpoint_initializer_135/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_135" op: "Identity" input: "checkpoint_initializer_135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_135" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias" input: "Identity_135" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_136/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_136/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/output/dense/kernel" } } } } node { name: "checkpoint_initializer_136/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_136" op: "RestoreV2" input: "checkpoint_initializer_136/prefix" input: "checkpoint_initializer_136/tensor_names" input: "checkpoint_initializer_136/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_136" op: "Identity" input: "checkpoint_initializer_136" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_136" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel" input: "Identity_136" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_137/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_137/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/self/key/bias" } } } } node { name: "checkpoint_initializer_137/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_137" op: "RestoreV2" input: "checkpoint_initializer_137/prefix" input: "checkpoint_initializer_137/tensor_names" input: "checkpoint_initializer_137/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_137" op: "Identity" input: "checkpoint_initializer_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_137" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias" input: "Identity_137" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_138/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_138/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/self/key/kernel" } } } } node { name: "checkpoint_initializer_138/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_138" op: "RestoreV2" input: "checkpoint_initializer_138/prefix" input: "checkpoint_initializer_138/tensor_names" input: "checkpoint_initializer_138/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_138" op: "Identity" input: "checkpoint_initializer_138" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_138" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel" input: "Identity_138" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_139/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_139/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/self/query/bias" } } } } node { name: "checkpoint_initializer_139/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_139" op: "RestoreV2" input: "checkpoint_initializer_139/prefix" input: "checkpoint_initializer_139/tensor_names" input: "checkpoint_initializer_139/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_139" op: "Identity" input: "checkpoint_initializer_139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_139" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias" input: "Identity_139" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_140/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_140/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/self/query/kernel" } } } } node { name: "checkpoint_initializer_140/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_140" op: "RestoreV2" input: "checkpoint_initializer_140/prefix" input: "checkpoint_initializer_140/tensor_names" input: "checkpoint_initializer_140/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_140" op: "Identity" input: "checkpoint_initializer_140" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_140" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel" input: "Identity_140" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_141/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_141/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/self/value/bias" } } } } node { name: "checkpoint_initializer_141/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_141" op: "RestoreV2" input: "checkpoint_initializer_141/prefix" input: "checkpoint_initializer_141/tensor_names" input: "checkpoint_initializer_141/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_141" op: "Identity" input: "checkpoint_initializer_141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_141" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias" input: "Identity_141" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_142/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_142/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/attention/self/value/kernel" } } } } node { name: "checkpoint_initializer_142/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_142" op: "RestoreV2" input: "checkpoint_initializer_142/prefix" input: "checkpoint_initializer_142/tensor_names" input: "checkpoint_initializer_142/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_142" op: "Identity" input: "checkpoint_initializer_142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_142" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel" input: "Identity_142" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_143/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_143/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/intermediate/dense/bias" } } } } node { name: "checkpoint_initializer_143/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_143" op: "RestoreV2" input: "checkpoint_initializer_143/prefix" input: "checkpoint_initializer_143/tensor_names" input: "checkpoint_initializer_143/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_143" op: "Identity" input: "checkpoint_initializer_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_143" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias" input: "Identity_143" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_144/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_144/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/intermediate/dense/kernel" } } } } node { name: "checkpoint_initializer_144/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_144" op: "RestoreV2" input: "checkpoint_initializer_144/prefix" input: "checkpoint_initializer_144/tensor_names" input: "checkpoint_initializer_144/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_144" op: "Identity" input: "checkpoint_initializer_144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_144" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel" input: "Identity_144" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_145/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_145/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/output/dense/bias" } } } } node { name: "checkpoint_initializer_145/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_145" op: "RestoreV2" input: "checkpoint_initializer_145/prefix" input: "checkpoint_initializer_145/tensor_names" input: "checkpoint_initializer_145/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_145" op: "Identity" input: "checkpoint_initializer_145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_145" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/bias" input: "Identity_145" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_146/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_146/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/encoder/layer_9/output/dense/kernel" } } } } node { name: "checkpoint_initializer_146/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_146" op: "RestoreV2" input: "checkpoint_initializer_146/prefix" input: "checkpoint_initializer_146/tensor_names" input: "checkpoint_initializer_146/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_146" op: "Identity" input: "checkpoint_initializer_146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_146" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/kernel" input: "Identity_146" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_147/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_147/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/pooler/dense/bias" } } } } node { name: "checkpoint_initializer_147/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_147" op: "RestoreV2" input: "checkpoint_initializer_147/prefix" input: "checkpoint_initializer_147/tensor_names" input: "checkpoint_initializer_147/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_147" op: "Identity" input: "checkpoint_initializer_147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_147" op: "AssignVariableOp" input: "bert/pooler/dense/bias" input: "Identity_147" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "checkpoint_initializer_148/prefix" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "/kaggle/input/chinese-roberta-wwm-ext-l-12-h-768-a-12/bert_model.ckpt" } } } } node { name: "checkpoint_initializer_148/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "bert/pooler/dense/kernel" } } } } node { name: "checkpoint_initializer_148/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 1 } } string_val: "" } } } } node { name: "checkpoint_initializer_148" op: "RestoreV2" input: "checkpoint_initializer_148/prefix" input: "checkpoint_initializer_148/tensor_names" input: "checkpoint_initializer_148/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtypes" value { list { type: DT_FLOAT } } } } node { name: "Identity_148" op: "Identity" input: "checkpoint_initializer_148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "AssignVariableOp_148" op: "AssignVariableOp" input: "bert/pooler/dense/kernel" input: "Identity_148" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999747378752e-06 } } } } node { name: "PolynomialDecay/Cast/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "PolynomialDecay/Cast_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "PolynomialDecay/Cast_2/ReadVariableOp" op: "ReadVariableOp" input: "global_step" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } } node { name: "PolynomialDecay/Cast_2" op: "Cast" input: "PolynomialDecay/Cast_2/ReadVariableOp" attr { key: "DstT" value { type: DT_FLOAT } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "PolynomialDecay/Cast_3/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 59292 } } } } node { name: "PolynomialDecay/Cast_3" op: "Cast" input: "PolynomialDecay/Cast_3/x" attr { key: "DstT" value { type: DT_FLOAT } } attr { key: "SrcT" value { type: DT_INT32 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "PolynomialDecay/Minimum" op: "Minimum" input: "PolynomialDecay/Cast_2" input: "PolynomialDecay/Cast_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "PolynomialDecay/truediv" op: "RealDiv" input: "PolynomialDecay/Minimum" input: "PolynomialDecay/Cast_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "PolynomialDecay/sub" op: "Sub" input: "Const_1" input: "PolynomialDecay/Cast/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "PolynomialDecay/sub_1/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "PolynomialDecay/sub_1" op: "Sub" input: "PolynomialDecay/sub_1/x" input: "PolynomialDecay/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "PolynomialDecay/Pow" op: "Pow" input: "PolynomialDecay/sub_1" input: "PolynomialDecay/Cast_1/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "PolynomialDecay/Mul" op: "Mul" input: "PolynomialDecay/sub" input: "PolynomialDecay/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "PolynomialDecay" op: "AddV2" input: "PolynomialDecay/Mul" input: "PolynomialDecay/Cast/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "Cast_1/ReadVariableOp" op: "ReadVariableOp" input: "global_step" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } } node { name: "Cast_1" op: "Cast" input: "Cast_1/ReadVariableOp" attr { key: "DstT" value { type: DT_INT32 } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "Const_2" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 5929 } } } } node { name: "Cast_2" op: "Cast" input: "Cast_1" attr { key: "DstT" value { type: DT_FLOAT } } attr { key: "SrcT" value { type: DT_INT32 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "Cast_3" op: "Cast" input: "Const_2" attr { key: "DstT" value { type: DT_FLOAT } } attr { key: "SrcT" value { type: DT_INT32 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "truediv" op: "RealDiv" input: "Cast_2" input: "Cast_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999747378752e-06 } } } } node { name: "mul" op: "Mul" input: "mul/x" input: "truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "Less" op: "Less" input: "Cast_1" input: "Const_2" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "Cast_4" op: "Cast" input: "Less" attr { key: "DstT" value { type: DT_FLOAT } } attr { key: "SrcT" value { type: DT_BOOL } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "sub/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "sub" op: "Sub" input: "sub/x" input: "Cast_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "mul_1" op: "Mul" input: "sub" input: "PolynomialDecay" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "mul_2" op: "Mul" input: "Cast_4" input: "mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "add" op: "AddV2" input: "mul_1" input: "mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/grad_ys_0/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/grad_ys_0" op: "Fill" input: "gradients/Shape" input: "gradients/grad_ys_0/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/loss/Mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/loss/Mean_grad/Reshape" op: "Reshape" input: "gradients/grad_ys_0" input: "gradients/loss/Mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/loss/Mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 16 } } } } node { name: "gradients/loss/Mean_grad/Tile" op: "Tile" input: "gradients/loss/Mean_grad/Reshape" input: "gradients/loss/Mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } } } } } } node { name: "gradients/loss/Mean_grad/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 16.0 } } } } node { name: "gradients/loss/Mean_grad/truediv" op: "RealDiv" input: "gradients/loss/Mean_grad/Tile" input: "gradients/loss/Mean_grad/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } } } } } } node { name: "gradients/loss/Neg_grad/Neg" op: "Neg" input: "gradients/loss/Mean_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } } } } } } node { name: "gradients/loss/Sum_grad/Maximum/x" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\020\000\000\000\001\000\000\000" } } } } node { name: "gradients/loss/Sum_grad/Maximum/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/loss/Sum_grad/Maximum" op: "Maximum" input: "gradients/loss/Sum_grad/Maximum/x" input: "gradients/loss/Sum_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/loss/Sum_grad/floordiv/x" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\020\000\000\000\003\000\000\000" } } } } node { name: "gradients/loss/Sum_grad/floordiv" op: "FloorDiv" input: "gradients/loss/Sum_grad/floordiv/x" input: "gradients/loss/Sum_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/loss/Sum_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\020\000\000\000\001\000\000\000" } } } } node { name: "gradients/loss/Sum_grad/Reshape" op: "Reshape" input: "gradients/loss/Neg_grad/Neg" input: "gradients/loss/Sum_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } } } } } } node { name: "gradients/loss/Sum_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\003\000\000\000" } } } } node { name: "gradients/loss/Sum_grad/Tile" op: "Tile" input: "gradients/loss/Sum_grad/Reshape" input: "gradients/loss/Sum_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 3 } } } } } } node { name: "gradients/loss/mul_grad/Mul" op: "Mul" input: "gradients/loss/Sum_grad/Tile" input: "loss/LogSoftmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 3 } } } } } } node { name: "gradients/loss/mul_grad/Mul_1" op: "Mul" input: "gradients/loss/Sum_grad/Tile" input: "loss/one_hot" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 3 } } } } } } node { name: "gradients/loss/LogSoftmax_grad/Exp" op: "Exp" input: "loss/LogSoftmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 3 } } } } } } node { name: "gradients/loss/LogSoftmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/loss/LogSoftmax_grad/Sum" op: "Sum" input: "gradients/loss/mul_grad/Mul_1" input: "gradients/loss/LogSoftmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/loss/LogSoftmax_grad/mul" op: "Mul" input: "gradients/loss/LogSoftmax_grad/Sum" input: "gradients/loss/LogSoftmax_grad/Exp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 3 } } } } } } node { name: "gradients/loss/LogSoftmax_grad/sub" op: "Sub" input: "gradients/loss/mul_grad/Mul_1" input: "gradients/loss/LogSoftmax_grad/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 3 } } } } } } node { name: "gradients/loss/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/loss/LogSoftmax_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/loss/MatMul_grad/MatMul" op: "MatMul" input: "gradients/loss/LogSoftmax_grad/sub" input: "loss/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/loss/MatMul_grad/MatMul_1" op: "MatMul" input: "gradients/loss/LogSoftmax_grad/sub" input: "loss/dropout/SelectV2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/loss/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/loss/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "loss/dropout/GreaterEqual" input: "gradients/loss/MatMul_grad/MatMul" input: "gradients/loss/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } } node { name: "gradients/loss/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\020\000\000\000\000\003\000\000" } } } } node { name: "gradients/loss/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\020\000\000\000\000\003\000\000" } } } } node { name: "gradients/loss/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/loss/dropout/SelectV2_grad/Shape" input: "gradients/loss/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/loss/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/loss/dropout/SelectV2_grad/SelectV2" input: "gradients/loss/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/loss/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/loss/dropout/SelectV2_grad/Sum" input: "gradients/loss/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } } node { name: "gradients/loss/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "loss/dropout/GreaterEqual" input: "gradients/loss/dropout/SelectV2_grad/zeros" input: "gradients/loss/MatMul_grad/MatMul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } } node { name: "gradients/loss/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/loss/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/loss/dropout/SelectV2_grad/Shape_2" input: "gradients/loss/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/loss/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/loss/dropout/SelectV2_grad/SelectV2_1" input: "gradients/loss/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/loss/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/loss/dropout/SelectV2_grad/Sum_1" input: "gradients/loss/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/loss/dropout/Mul_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\020\000\000\000\000\003\000\000" } } } } node { name: "gradients/loss/dropout/Mul_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/loss/dropout/Mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/loss/dropout/Mul_grad/BroadcastGradientArgs/s0" input: "gradients/loss/dropout/Mul_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/loss/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/loss/dropout/SelectV2_grad/Reshape" input: "loss/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } } node { name: "gradients/loss/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/pooler/dense/Tanh" input: "gradients/loss/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } } node { name: "gradients/loss/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/loss/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/loss/dropout/Mul_grad/Mul_1" input: "gradients/loss/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/loss/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/loss/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/loss/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/loss/dropout/Mul_grad/Sum" input: "gradients/loss/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/pooler/dense/Tanh_grad/TanhGrad" op: "TanhGrad" input: "bert/pooler/dense/Tanh" input: "gradients/loss/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } } node { name: "gradients/bert/pooler/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/pooler/dense/Tanh_grad/TanhGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/pooler/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/pooler/dense/Tanh_grad/TanhGrad" input: "bert/pooler/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/pooler/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/pooler/Squeeze" input: "gradients/bert/pooler/dense/Tanh_grad/TanhGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/pooler/Squeeze_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/pooler/Squeeze_grad/Reshape" op: "Reshape" input: "gradients/bert/pooler/dense/MatMul_grad/MatMul" input: "gradients/bert/pooler/Squeeze_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 768 } } } } } } node { name: "gradients/bert/pooler/strided_slice_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "gradients/bert/pooler/strided_slice_grad/StridedSliceGrad/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "gradients/bert/pooler/strided_slice_grad/StridedSliceGrad/begin" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\000\000\000\000\000\000\000\000\000\000\000\000" } } } } node { name: "gradients/bert/pooler/strided_slice_grad/StridedSliceGrad/end" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\000\000\000\000\001\000\000\000\000\000\000\000" } } } } node { name: "gradients/bert/pooler/strided_slice_grad/StridedSliceGrad/strides" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\001\000\000\000\001\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/pooler/strided_slice_grad/StridedSliceGrad" op: "StridedSliceGrad" input: "gradients/bert/pooler/strided_slice_grad/StridedSliceGrad/shape" input: "gradients/bert/pooler/strided_slice_grad/StridedSliceGrad/begin" input: "gradients/bert/pooler/strided_slice_grad/StridedSliceGrad/end" input: "gradients/bert/pooler/strided_slice_grad/StridedSliceGrad/strides" input: "gradients/bert/pooler/Squeeze_grad/Reshape" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } attr { key: "begin_mask" value { i: 5 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 5 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "gradients/bert/encoder/Reshape_13_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/Reshape_13_grad/Reshape" op: "Reshape" input: "gradients/bert/pooler/strided_slice_grad/StridedSliceGrad" input: "gradients/bert/encoder/Reshape_13_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/Reshape_13_grad/Reshape" input: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/Reshape_13_grad/Reshape" input: "bert/encoder/layer_11/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/BroadcastGradientArgs/s0" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/bert/encoder/Reshape_13_grad/Reshape" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/encoder/Reshape_13_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/BroadcastGradientArgs/s0" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/output/layer_normalization_24/moments/mean" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN" op: "AddN" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/BroadcastGradientArgs/s0" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN" input: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/Rsqrt" input: "gradients/AddN" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_11/output/layer_normalization_24/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add_grad/BroadcastGradientArgs/s0" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/Maximum/x" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/Maximum/x" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/floordiv/x" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/floordiv/x" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_11/output/add" input: "bert/encoder/layer_11/output/layer_normalization_24/moments/StopGradient" input: "^gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/BroadcastGradientArgs/s0" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_1" op: "AddN" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_11/output/dropout/GreaterEqual" input: "gradients/AddN_1" input: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_11/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/BroadcastGradientArgs/s0" input: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_11/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_11/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_11/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_11/intermediate/dense/mul_3" input: "gradients/bert/encoder/layer_11/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_3_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_11/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_3_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_11/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/BroadcastGradientArgs/s0_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/BroadcastGradientArgs/s0_1" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_3_grad/Mul_1" input: "bert/encoder/layer_11/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/intermediate/dense/mul_2/x" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_3_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/add_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/add_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_11/intermediate/dense/add_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/add_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/add_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/add_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/intermediate/dense/add_1_grad/Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/add_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Tanh_grad/TanhGrad" op: "TanhGrad" input: "bert/encoder/layer_11/intermediate/dense/Tanh" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_2_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/Tanh_grad/TanhGrad" input: "bert/encoder/layer_11/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/intermediate/dense/mul_1/x" input: "gradients/bert/encoder/layer_11/intermediate/dense/Tanh_grad/TanhGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_11/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/BroadcastGradientArgs/s0" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_11/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/sub/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/sub" op: "Sub" input: "bert/encoder/layer_11/intermediate/dense/Pow/y" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/sub/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Pow" op: "Pow" input: "bert/encoder/layer_11/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Greater/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Greater" op: "Greater" input: "bert/encoder/layer_11/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Greater/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/ones_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/ones_like" op: "Fill" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/ones_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Select" op: "Select" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Greater" input: "bert/encoder/layer_11/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/ones_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Log" op: "Log" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Select" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/zeros_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/zeros_like" op: "Fill" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/zeros_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Select_1" op: "Select" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Greater" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Log" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/zeros_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/mul_2" op: "Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_11/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/mul_3" op: "Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/mul_2" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Select_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/mul_3" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Sum" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_2" op: "AddN" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_3_grad/Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_11/intermediate/dense/Pow_grad/mul_1" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/intermediate/dense/mul_3_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_2" input: "bert/encoder/layer_11/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/add_1" input: "gradients/AddN_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_3" op: "AddN" input: "gradients/AddN_1" input: "gradients/bert/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_3" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_3" input: "bert/encoder/layer_11/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_3" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_4" op: "AddN" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_4" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/Rsqrt" input: "gradients/AddN_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_11/attention/output/add" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/moments/StopGradient" input: "^gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_5" op: "AddN" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_11/attention/output/dropout/GreaterEqual" input: "gradients/AddN_5" input: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_11/attention/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_11/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/attention/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_11/attention/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_11/attention/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_11/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_11/attention/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_11/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_11/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_11/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_11/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_11/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_11/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_11/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_11/attention/self/dropout/SelectV2" input: "gradients/bert/encoder/layer_11/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_11/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_11/attention/self/MatMul_1_grad/MatMul" input: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_11/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/zeros" input: "gradients/bert/encoder/layer_11/attention/self/MatMul_1_grad/MatMul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_11/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_11/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_11/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/BroadcastGradientArgs/s0" input: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_11/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/attention/self/Softmax" input: "gradients/bert/encoder/layer_11/attention/self/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/Mul" input: "bert/encoder/layer_11/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_11/attention/self/dropout/Mul_grad/Mul" input: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_11/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\001\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs/s0" input: "gradients/bert/encoder/layer_11/attention/self/add_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_11/attention/self/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\001\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/self/add_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_11/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_1" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul_1" input: "bert/encoder/layer_11/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_11/attention/self/MatMul" input: "gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/MatMul_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_11/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_11/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_11/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_11/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_11/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_11/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_11/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_11/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_11/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_11/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_11/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_11/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_1" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_11/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_11/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_1" input: "gradients/bert/encoder/layer_11/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_6" op: "AddN" input: "gradients/AddN_5" input: "gradients/bert/encoder/layer_11/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_11/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_11/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_6" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_6" input: "bert/encoder/layer_10/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_6" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/output/layer_normalization_22/moments/mean" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_7" op: "AddN" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_7" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/Rsqrt" input: "gradients/AddN_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_10/output/layer_normalization_22/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_10/output/add" input: "bert/encoder/layer_10/output/layer_normalization_22/moments/StopGradient" input: "^gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_8" op: "AddN" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_10/output/dropout/GreaterEqual" input: "gradients/AddN_8" input: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_10/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_10/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_10/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_10/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_10/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_10/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_10/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_10/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_10/intermediate/dense/mul_3" input: "gradients/bert/encoder/layer_10/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_3_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_10/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_3_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_10/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_3_grad/Mul_1" input: "bert/encoder/layer_10/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_2_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_2_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/intermediate/dense/mul_2/x" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_3_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/add_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/add_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_10/intermediate/dense/add_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/add_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/add_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/add_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/intermediate/dense/add_1_grad/Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/add_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Tanh_grad/TanhGrad" op: "TanhGrad" input: "bert/encoder/layer_10/intermediate/dense/Tanh" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_2_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/Tanh_grad/TanhGrad" input: "bert/encoder/layer_10/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/intermediate/dense/mul_1/x" input: "gradients/bert/encoder/layer_10/intermediate/dense/Tanh_grad/TanhGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_10/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_10/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/sub/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/sub" op: "Sub" input: "bert/encoder/layer_10/intermediate/dense/Pow/y" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/sub/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Pow" op: "Pow" input: "bert/encoder/layer_10/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Greater/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Greater" op: "Greater" input: "bert/encoder/layer_10/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Greater/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/ones_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/ones_like" op: "Fill" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/ones_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Select" op: "Select" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Greater" input: "bert/encoder/layer_10/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/ones_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Log" op: "Log" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Select" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/zeros_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/zeros_like" op: "Fill" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/zeros_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Select_1" op: "Select" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Greater" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Log" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/zeros_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/mul_2" op: "Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_10/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/mul_3" op: "Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/mul_2" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Select_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/mul_3" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Sum" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_9" op: "AddN" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_3_grad/Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_10/intermediate/dense/Pow_grad/mul_1" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/intermediate/dense/mul_3_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_9" input: "bert/encoder/layer_10/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/add_1" input: "gradients/AddN_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_10" op: "AddN" input: "gradients/AddN_8" input: "gradients/bert/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_10" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_10" input: "bert/encoder/layer_10/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_10" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_11" op: "AddN" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_11" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/Rsqrt" input: "gradients/AddN_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_10/attention/output/add" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/moments/StopGradient" input: "^gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_12" op: "AddN" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_10/attention/output/dropout/GreaterEqual" input: "gradients/AddN_12" input: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_10/attention/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_10/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/attention/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_10/attention/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_10/attention/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_10/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_10/attention/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_10/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_10/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_10/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_10/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_10/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_10/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_10/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_10/attention/self/dropout/SelectV2" input: "gradients/bert/encoder/layer_10/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_10/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_10/attention/self/MatMul_1_grad/MatMul" input: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_10/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/zeros" input: "gradients/bert/encoder/layer_10/attention/self/MatMul_1_grad/MatMul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_10/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_10/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_10/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_10/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/attention/self/Softmax" input: "gradients/bert/encoder/layer_10/attention/self/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/self/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_10/attention/self/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/self/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/dropout/Mul_grad/Mul" input: "bert/encoder/layer_10/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_10/attention/self/dropout/Mul_grad/Mul" input: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_10/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_10/attention/self/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_10/attention/self/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\001\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/self/add_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_10/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_1" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul_1" input: "bert/encoder/layer_10/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_10/attention/self/MatMul" input: "gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/MatMul_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_10/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_10/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_10/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_10/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_10/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_10/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_10/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_10/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_10/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_10/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_10/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_10/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_1" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_10/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_10/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_1" input: "gradients/bert/encoder/layer_10/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_13" op: "AddN" input: "gradients/AddN_12" input: "gradients/bert/encoder/layer_10/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_10/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_10/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_13" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_13" input: "bert/encoder/layer_9/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_13" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/output/layer_normalization_20/moments/mean" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_14" op: "AddN" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_14" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/Rsqrt" input: "gradients/AddN_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_9/output/layer_normalization_20/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_9/output/add" input: "bert/encoder/layer_9/output/layer_normalization_20/moments/StopGradient" input: "^gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_15" op: "AddN" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_9/output/dropout/GreaterEqual" input: "gradients/AddN_15" input: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_9/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_9/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_9/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_9/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_9/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_9/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_9/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_9/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_9/intermediate/dense/mul_3" input: "gradients/bert/encoder/layer_9/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_3_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_9/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_3_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_9/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_3_grad/Mul_1" input: "bert/encoder/layer_9/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_2_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_2_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/intermediate/dense/mul_2/x" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_3_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/add_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/add_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_9/intermediate/dense/add_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/add_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/add_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/add_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/intermediate/dense/add_1_grad/Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/add_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Tanh_grad/TanhGrad" op: "TanhGrad" input: "bert/encoder/layer_9/intermediate/dense/Tanh" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_2_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/Tanh_grad/TanhGrad" input: "bert/encoder/layer_9/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/intermediate/dense/mul_1/x" input: "gradients/bert/encoder/layer_9/intermediate/dense/Tanh_grad/TanhGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_9/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_9/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/sub/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/sub" op: "Sub" input: "bert/encoder/layer_9/intermediate/dense/Pow/y" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/sub/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Pow" op: "Pow" input: "bert/encoder/layer_9/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Greater/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Greater" op: "Greater" input: "bert/encoder/layer_9/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Greater/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/ones_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/ones_like" op: "Fill" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/ones_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Select" op: "Select" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Greater" input: "bert/encoder/layer_9/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/ones_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Log" op: "Log" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Select" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/zeros_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/zeros_like" op: "Fill" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/zeros_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Select_1" op: "Select" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Greater" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Log" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/zeros_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/mul_2" op: "Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_9/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/mul_3" op: "Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/mul_2" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Select_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/mul_3" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Sum" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_16" op: "AddN" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_3_grad/Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_9/intermediate/dense/Pow_grad/mul_1" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/intermediate/dense/mul_3_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_16" input: "bert/encoder/layer_9/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/add_1" input: "gradients/AddN_16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_17" op: "AddN" input: "gradients/AddN_15" input: "gradients/bert/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_17" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_17" input: "bert/encoder/layer_9/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_17" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_18" op: "AddN" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_18" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/Rsqrt" input: "gradients/AddN_18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_9/attention/output/add" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/moments/StopGradient" input: "^gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_19" op: "AddN" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_9/attention/output/dropout/GreaterEqual" input: "gradients/AddN_19" input: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_9/attention/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_9/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/attention/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_9/attention/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_9/attention/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_9/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_9/attention/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_9/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_9/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_9/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_9/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_9/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_9/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_9/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_9/attention/self/dropout/SelectV2" input: "gradients/bert/encoder/layer_9/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_9/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_9/attention/self/MatMul_1_grad/MatMul" input: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_9/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/zeros" input: "gradients/bert/encoder/layer_9/attention/self/MatMul_1_grad/MatMul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_9/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_9/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_9/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_9/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/attention/self/Softmax" input: "gradients/bert/encoder/layer_9/attention/self/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/self/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_9/attention/self/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/self/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/dropout/Mul_grad/Mul" input: "bert/encoder/layer_9/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_9/attention/self/dropout/Mul_grad/Mul" input: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_9/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_9/attention/self/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_9/attention/self/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\001\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/self/add_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_9/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_1" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul_1" input: "bert/encoder/layer_9/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_9/attention/self/MatMul" input: "gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/MatMul_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_9/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_9/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_9/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_9/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_9/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_9/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_9/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_9/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_9/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_9/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_9/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_9/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_1" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_9/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_9/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_1" input: "gradients/bert/encoder/layer_9/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_20" op: "AddN" input: "gradients/AddN_19" input: "gradients/bert/encoder/layer_9/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_9/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_9/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_20" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_20" input: "bert/encoder/layer_8/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_20" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/output/layer_normalization_18/moments/mean" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_21" op: "AddN" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_21" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/Rsqrt" input: "gradients/AddN_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_8/output/layer_normalization_18/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_8/output/add" input: "bert/encoder/layer_8/output/layer_normalization_18/moments/StopGradient" input: "^gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_22" op: "AddN" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_8/output/dropout/GreaterEqual" input: "gradients/AddN_22" input: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_8/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_8/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_8/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_8/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_8/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_8/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_8/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_8/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_8/intermediate/dense/mul_3" input: "gradients/bert/encoder/layer_8/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_3_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_8/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_3_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_8/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_3_grad/Mul_1" input: "bert/encoder/layer_8/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_2_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_2_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/intermediate/dense/mul_2/x" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_3_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/add_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/add_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_8/intermediate/dense/add_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/add_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/add_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/add_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/intermediate/dense/add_1_grad/Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/add_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Tanh_grad/TanhGrad" op: "TanhGrad" input: "bert/encoder/layer_8/intermediate/dense/Tanh" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_2_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/Tanh_grad/TanhGrad" input: "bert/encoder/layer_8/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/intermediate/dense/mul_1/x" input: "gradients/bert/encoder/layer_8/intermediate/dense/Tanh_grad/TanhGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_8/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_8/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/sub/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/sub" op: "Sub" input: "bert/encoder/layer_8/intermediate/dense/Pow/y" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/sub/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Pow" op: "Pow" input: "bert/encoder/layer_8/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Greater/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Greater" op: "Greater" input: "bert/encoder/layer_8/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Greater/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/ones_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/ones_like" op: "Fill" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/ones_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Select" op: "Select" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Greater" input: "bert/encoder/layer_8/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/ones_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Log" op: "Log" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Select" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/zeros_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/zeros_like" op: "Fill" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/zeros_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Select_1" op: "Select" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Greater" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Log" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/zeros_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/mul_2" op: "Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_8/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/mul_3" op: "Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/mul_2" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Select_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/mul_3" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Sum" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_23" op: "AddN" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_3_grad/Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_8/intermediate/dense/Pow_grad/mul_1" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/intermediate/dense/mul_3_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_23" input: "bert/encoder/layer_8/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/add_1" input: "gradients/AddN_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_24" op: "AddN" input: "gradients/AddN_22" input: "gradients/bert/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_24" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_24" input: "bert/encoder/layer_8/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_24" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_25" op: "AddN" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_25" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/Rsqrt" input: "gradients/AddN_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_8/attention/output/add" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/moments/StopGradient" input: "^gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_26" op: "AddN" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_8/attention/output/dropout/GreaterEqual" input: "gradients/AddN_26" input: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_8/attention/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_8/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/attention/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_8/attention/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_8/attention/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_8/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_8/attention/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_8/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_8/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_8/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_8/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_8/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_8/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_8/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_8/attention/self/dropout/SelectV2" input: "gradients/bert/encoder/layer_8/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_8/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_8/attention/self/MatMul_1_grad/MatMul" input: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_8/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/zeros" input: "gradients/bert/encoder/layer_8/attention/self/MatMul_1_grad/MatMul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_8/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_8/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_8/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_8/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/attention/self/Softmax" input: "gradients/bert/encoder/layer_8/attention/self/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/self/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_8/attention/self/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/self/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/dropout/Mul_grad/Mul" input: "bert/encoder/layer_8/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_8/attention/self/dropout/Mul_grad/Mul" input: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_8/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_8/attention/self/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_8/attention/self/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\001\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/self/add_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_8/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_1" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul_1" input: "bert/encoder/layer_8/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_8/attention/self/MatMul" input: "gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/MatMul_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_8/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_8/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_8/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_8/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_8/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_8/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_8/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_8/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_8/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_8/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_8/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_8/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_1" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_8/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_8/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_1" input: "gradients/bert/encoder/layer_8/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_27" op: "AddN" input: "gradients/AddN_26" input: "gradients/bert/encoder/layer_8/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_8/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_8/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_27" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_27" input: "bert/encoder/layer_7/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_27" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/output/layer_normalization_16/moments/mean" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_28" op: "AddN" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_28" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/Rsqrt" input: "gradients/AddN_28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_7/output/layer_normalization_16/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_7/output/add" input: "bert/encoder/layer_7/output/layer_normalization_16/moments/StopGradient" input: "^gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_29" op: "AddN" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_7/output/dropout/GreaterEqual" input: "gradients/AddN_29" input: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_7/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_7/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_7/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_7/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_7/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_7/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_7/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_7/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_7/intermediate/dense/mul_3" input: "gradients/bert/encoder/layer_7/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_3_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_7/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_3_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_7/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_3_grad/Mul_1" input: "bert/encoder/layer_7/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_2_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_2_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/intermediate/dense/mul_2/x" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_3_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/add_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/add_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_7/intermediate/dense/add_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/add_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/add_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/add_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/intermediate/dense/add_1_grad/Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/add_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Tanh_grad/TanhGrad" op: "TanhGrad" input: "bert/encoder/layer_7/intermediate/dense/Tanh" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_2_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/Tanh_grad/TanhGrad" input: "bert/encoder/layer_7/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/intermediate/dense/mul_1/x" input: "gradients/bert/encoder/layer_7/intermediate/dense/Tanh_grad/TanhGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_7/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_7/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/sub/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/sub" op: "Sub" input: "bert/encoder/layer_7/intermediate/dense/Pow/y" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/sub/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Pow" op: "Pow" input: "bert/encoder/layer_7/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Greater/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Greater" op: "Greater" input: "bert/encoder/layer_7/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Greater/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/ones_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/ones_like" op: "Fill" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/ones_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Select" op: "Select" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Greater" input: "bert/encoder/layer_7/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/ones_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Log" op: "Log" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Select" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/zeros_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/zeros_like" op: "Fill" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/zeros_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Select_1" op: "Select" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Greater" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Log" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/zeros_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/mul_2" op: "Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_7/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/mul_3" op: "Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/mul_2" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Select_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/mul_3" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Sum" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_30" op: "AddN" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_3_grad/Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_7/intermediate/dense/Pow_grad/mul_1" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/intermediate/dense/mul_3_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_30" input: "bert/encoder/layer_7/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/add_1" input: "gradients/AddN_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_31" op: "AddN" input: "gradients/AddN_29" input: "gradients/bert/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_31" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_31" input: "bert/encoder/layer_7/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_31" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_32" op: "AddN" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_32" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/Rsqrt" input: "gradients/AddN_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_7/attention/output/add" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/moments/StopGradient" input: "^gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_33" op: "AddN" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_7/attention/output/dropout/GreaterEqual" input: "gradients/AddN_33" input: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_7/attention/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_7/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/attention/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_7/attention/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_7/attention/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_7/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_7/attention/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_7/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_7/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_7/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_7/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_7/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_7/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_7/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_7/attention/self/dropout/SelectV2" input: "gradients/bert/encoder/layer_7/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_7/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_7/attention/self/MatMul_1_grad/MatMul" input: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_7/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/zeros" input: "gradients/bert/encoder/layer_7/attention/self/MatMul_1_grad/MatMul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_7/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_7/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_7/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_7/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/attention/self/Softmax" input: "gradients/bert/encoder/layer_7/attention/self/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/self/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_7/attention/self/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/self/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/dropout/Mul_grad/Mul" input: "bert/encoder/layer_7/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_7/attention/self/dropout/Mul_grad/Mul" input: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_7/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_7/attention/self/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_7/attention/self/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\001\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/self/add_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_7/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_1" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul_1" input: "bert/encoder/layer_7/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_7/attention/self/MatMul" input: "gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/MatMul_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_7/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_7/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_7/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_7/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_7/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_7/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_7/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_7/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_7/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_7/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_7/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_7/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_1" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_7/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_7/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_1" input: "gradients/bert/encoder/layer_7/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_34" op: "AddN" input: "gradients/AddN_33" input: "gradients/bert/encoder/layer_7/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_7/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_7/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_34" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_34" input: "bert/encoder/layer_6/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_34" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/output/layer_normalization_14/moments/mean" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_35" op: "AddN" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_35" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/Rsqrt" input: "gradients/AddN_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_6/output/layer_normalization_14/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_6/output/add" input: "bert/encoder/layer_6/output/layer_normalization_14/moments/StopGradient" input: "^gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_36" op: "AddN" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_6/output/dropout/GreaterEqual" input: "gradients/AddN_36" input: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_6/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_6/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_6/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_6/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_6/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_6/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_6/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_6/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_6/intermediate/dense/mul_3" input: "gradients/bert/encoder/layer_6/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_3_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_6/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_3_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_6/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_3_grad/Mul_1" input: "bert/encoder/layer_6/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_2_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_2_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/intermediate/dense/mul_2/x" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_3_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/add_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/add_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_6/intermediate/dense/add_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/add_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/add_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/add_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/intermediate/dense/add_1_grad/Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/add_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Tanh_grad/TanhGrad" op: "TanhGrad" input: "bert/encoder/layer_6/intermediate/dense/Tanh" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_2_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/Tanh_grad/TanhGrad" input: "bert/encoder/layer_6/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/intermediate/dense/mul_1/x" input: "gradients/bert/encoder/layer_6/intermediate/dense/Tanh_grad/TanhGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_6/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_6/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/sub/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/sub" op: "Sub" input: "bert/encoder/layer_6/intermediate/dense/Pow/y" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/sub/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Pow" op: "Pow" input: "bert/encoder/layer_6/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Greater/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Greater" op: "Greater" input: "bert/encoder/layer_6/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Greater/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/ones_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/ones_like" op: "Fill" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/ones_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Select" op: "Select" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Greater" input: "bert/encoder/layer_6/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/ones_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Log" op: "Log" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Select" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/zeros_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/zeros_like" op: "Fill" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/zeros_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Select_1" op: "Select" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Greater" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Log" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/zeros_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/mul_2" op: "Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_6/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/mul_3" op: "Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/mul_2" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Select_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/mul_3" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Sum" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_37" op: "AddN" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_3_grad/Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_6/intermediate/dense/Pow_grad/mul_1" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/intermediate/dense/mul_3_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_37" input: "bert/encoder/layer_6/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/add_1" input: "gradients/AddN_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_38" op: "AddN" input: "gradients/AddN_36" input: "gradients/bert/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_38" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_38" input: "bert/encoder/layer_6/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_38" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_39" op: "AddN" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_39" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/Rsqrt" input: "gradients/AddN_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_6/attention/output/add" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/moments/StopGradient" input: "^gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_40" op: "AddN" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_6/attention/output/dropout/GreaterEqual" input: "gradients/AddN_40" input: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_6/attention/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_6/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/attention/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_6/attention/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_6/attention/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_6/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_6/attention/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_6/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_6/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_6/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_6/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_6/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_6/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_6/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_6/attention/self/dropout/SelectV2" input: "gradients/bert/encoder/layer_6/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_6/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_6/attention/self/MatMul_1_grad/MatMul" input: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_6/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/zeros" input: "gradients/bert/encoder/layer_6/attention/self/MatMul_1_grad/MatMul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_6/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_6/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_6/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_6/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/attention/self/Softmax" input: "gradients/bert/encoder/layer_6/attention/self/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/self/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_6/attention/self/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/self/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/dropout/Mul_grad/Mul" input: "bert/encoder/layer_6/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_6/attention/self/dropout/Mul_grad/Mul" input: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_6/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_6/attention/self/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_6/attention/self/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\001\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/self/add_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_6/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_1" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul_1" input: "bert/encoder/layer_6/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_6/attention/self/MatMul" input: "gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/MatMul_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_6/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_6/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_6/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_6/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_6/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_6/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_6/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_6/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_6/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_6/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_6/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_6/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_1" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_6/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_6/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_1" input: "gradients/bert/encoder/layer_6/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_41" op: "AddN" input: "gradients/AddN_40" input: "gradients/bert/encoder/layer_6/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_6/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_6/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_41" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_41" input: "bert/encoder/layer_5/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_41" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/output/layer_normalization_12/moments/mean" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_42" op: "AddN" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_42" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/Rsqrt" input: "gradients/AddN_42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_5/output/layer_normalization_12/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_5/output/add" input: "bert/encoder/layer_5/output/layer_normalization_12/moments/StopGradient" input: "^gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_43" op: "AddN" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_5/output/dropout/GreaterEqual" input: "gradients/AddN_43" input: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_5/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_5/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_5/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_5/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_5/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_5/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_5/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_5/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_5/intermediate/dense/mul_3" input: "gradients/bert/encoder/layer_5/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_3_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_5/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_3_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_5/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_3_grad/Mul_1" input: "bert/encoder/layer_5/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_2_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_2_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/intermediate/dense/mul_2/x" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_3_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/add_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/add_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_5/intermediate/dense/add_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/add_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/add_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/add_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/intermediate/dense/add_1_grad/Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/add_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Tanh_grad/TanhGrad" op: "TanhGrad" input: "bert/encoder/layer_5/intermediate/dense/Tanh" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_2_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/Tanh_grad/TanhGrad" input: "bert/encoder/layer_5/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/intermediate/dense/mul_1/x" input: "gradients/bert/encoder/layer_5/intermediate/dense/Tanh_grad/TanhGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_5/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_5/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/sub/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/sub" op: "Sub" input: "bert/encoder/layer_5/intermediate/dense/Pow/y" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/sub/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Pow" op: "Pow" input: "bert/encoder/layer_5/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Greater/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Greater" op: "Greater" input: "bert/encoder/layer_5/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Greater/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/ones_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/ones_like" op: "Fill" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/ones_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Select" op: "Select" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Greater" input: "bert/encoder/layer_5/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/ones_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Log" op: "Log" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Select" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/zeros_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/zeros_like" op: "Fill" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/zeros_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Select_1" op: "Select" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Greater" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Log" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/zeros_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/mul_2" op: "Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_5/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/mul_3" op: "Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/mul_2" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Select_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/mul_3" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Sum" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_44" op: "AddN" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_3_grad/Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_5/intermediate/dense/Pow_grad/mul_1" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/intermediate/dense/mul_3_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_44" input: "bert/encoder/layer_5/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/add_1" input: "gradients/AddN_44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_45" op: "AddN" input: "gradients/AddN_43" input: "gradients/bert/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_45" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_45" input: "bert/encoder/layer_5/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_45" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_46" op: "AddN" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_46" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/Rsqrt" input: "gradients/AddN_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_5/attention/output/add" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/moments/StopGradient" input: "^gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_47" op: "AddN" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_5/attention/output/dropout/GreaterEqual" input: "gradients/AddN_47" input: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_5/attention/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_5/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/attention/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_5/attention/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_5/attention/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_5/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_5/attention/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_5/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_5/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_5/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_5/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_5/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_5/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_5/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_5/attention/self/dropout/SelectV2" input: "gradients/bert/encoder/layer_5/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_5/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_5/attention/self/MatMul_1_grad/MatMul" input: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_5/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/zeros" input: "gradients/bert/encoder/layer_5/attention/self/MatMul_1_grad/MatMul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_5/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_5/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_5/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_5/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/attention/self/Softmax" input: "gradients/bert/encoder/layer_5/attention/self/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/self/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_5/attention/self/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/self/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/dropout/Mul_grad/Mul" input: "bert/encoder/layer_5/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_5/attention/self/dropout/Mul_grad/Mul" input: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_5/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_5/attention/self/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_5/attention/self/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\001\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/self/add_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_5/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_1" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul_1" input: "bert/encoder/layer_5/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_5/attention/self/MatMul" input: "gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/MatMul_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_5/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_5/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_5/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_5/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_5/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_5/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_5/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_5/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_5/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_5/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_5/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_5/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_1" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_5/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_5/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_1" input: "gradients/bert/encoder/layer_5/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_48" op: "AddN" input: "gradients/AddN_47" input: "gradients/bert/encoder/layer_5/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_5/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_5/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_48" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_48" input: "bert/encoder/layer_4/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_48" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/output/layer_normalization_10/moments/mean" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_49" op: "AddN" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_49" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/Rsqrt" input: "gradients/AddN_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_4/output/layer_normalization_10/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_4/output/add" input: "bert/encoder/layer_4/output/layer_normalization_10/moments/StopGradient" input: "^gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_50" op: "AddN" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_4/output/dropout/GreaterEqual" input: "gradients/AddN_50" input: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_4/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_4/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_4/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_4/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_4/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_4/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_4/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_4/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_4/intermediate/dense/mul_3" input: "gradients/bert/encoder/layer_4/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_3_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_4/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_3_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_4/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_3_grad/Mul_1" input: "bert/encoder/layer_4/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_2_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_2_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/intermediate/dense/mul_2/x" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_3_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/add_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/add_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_4/intermediate/dense/add_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/add_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/add_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/add_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/intermediate/dense/add_1_grad/Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/add_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Tanh_grad/TanhGrad" op: "TanhGrad" input: "bert/encoder/layer_4/intermediate/dense/Tanh" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_2_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/Tanh_grad/TanhGrad" input: "bert/encoder/layer_4/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/intermediate/dense/mul_1/x" input: "gradients/bert/encoder/layer_4/intermediate/dense/Tanh_grad/TanhGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_4/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_4/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/sub/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/sub" op: "Sub" input: "bert/encoder/layer_4/intermediate/dense/Pow/y" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/sub/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Pow" op: "Pow" input: "bert/encoder/layer_4/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Greater/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Greater" op: "Greater" input: "bert/encoder/layer_4/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Greater/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/ones_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/ones_like" op: "Fill" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/ones_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Select" op: "Select" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Greater" input: "bert/encoder/layer_4/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/ones_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Log" op: "Log" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Select" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/zeros_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/zeros_like" op: "Fill" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/zeros_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Select_1" op: "Select" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Greater" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Log" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/zeros_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/mul_2" op: "Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_4/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/mul_3" op: "Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/mul_2" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Select_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/mul_3" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Sum" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_51" op: "AddN" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_3_grad/Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_4/intermediate/dense/Pow_grad/mul_1" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/intermediate/dense/mul_3_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_51" input: "bert/encoder/layer_4/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/add_1" input: "gradients/AddN_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_52" op: "AddN" input: "gradients/AddN_50" input: "gradients/bert/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_52" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_52" input: "bert/encoder/layer_4/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_52" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_53" op: "AddN" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_53" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/Rsqrt" input: "gradients/AddN_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_4/attention/output/add" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/moments/StopGradient" input: "^gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_54" op: "AddN" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_4/attention/output/dropout/GreaterEqual" input: "gradients/AddN_54" input: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_4/attention/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_4/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/attention/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_4/attention/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_4/attention/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_4/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_4/attention/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_4/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_4/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_4/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_4/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_4/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_4/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_4/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_4/attention/self/dropout/SelectV2" input: "gradients/bert/encoder/layer_4/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_4/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_4/attention/self/MatMul_1_grad/MatMul" input: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_4/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/zeros" input: "gradients/bert/encoder/layer_4/attention/self/MatMul_1_grad/MatMul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_4/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_4/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_4/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_4/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/attention/self/Softmax" input: "gradients/bert/encoder/layer_4/attention/self/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/self/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_4/attention/self/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/self/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/dropout/Mul_grad/Mul" input: "bert/encoder/layer_4/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_4/attention/self/dropout/Mul_grad/Mul" input: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_4/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_4/attention/self/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_4/attention/self/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\001\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/self/add_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_4/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_1" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul_1" input: "bert/encoder/layer_4/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_4/attention/self/MatMul" input: "gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/MatMul_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_4/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_4/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_4/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_4/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_4/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_4/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_4/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_4/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_4/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_4/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_4/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_4/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_1" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_4/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_4/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_1" input: "gradients/bert/encoder/layer_4/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_55" op: "AddN" input: "gradients/AddN_54" input: "gradients/bert/encoder/layer_4/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_4/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_4/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_55" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_55" input: "bert/encoder/layer_3/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_55" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/output/layer_normalization_8/moments/mean" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_56" op: "AddN" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_56" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/Rsqrt" input: "gradients/AddN_56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_3/output/layer_normalization_8/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_3/output/add" input: "bert/encoder/layer_3/output/layer_normalization_8/moments/StopGradient" input: "^gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_57" op: "AddN" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_3/output/dropout/GreaterEqual" input: "gradients/AddN_57" input: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_3/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_3/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_3/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_3/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_3/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_3/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_3/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_3/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_3/intermediate/dense/mul_3" input: "gradients/bert/encoder/layer_3/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_3_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_3/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_3_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_3/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_3_grad/Mul_1" input: "bert/encoder/layer_3/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_2_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_2_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/intermediate/dense/mul_2/x" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_3_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/add_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/add_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_3/intermediate/dense/add_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/add_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/add_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/add_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/intermediate/dense/add_1_grad/Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/add_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Tanh_grad/TanhGrad" op: "TanhGrad" input: "bert/encoder/layer_3/intermediate/dense/Tanh" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_2_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/Tanh_grad/TanhGrad" input: "bert/encoder/layer_3/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/intermediate/dense/mul_1/x" input: "gradients/bert/encoder/layer_3/intermediate/dense/Tanh_grad/TanhGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_3/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_3/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/sub/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/sub" op: "Sub" input: "bert/encoder/layer_3/intermediate/dense/Pow/y" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/sub/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Pow" op: "Pow" input: "bert/encoder/layer_3/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Greater/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Greater" op: "Greater" input: "bert/encoder/layer_3/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Greater/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/ones_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/ones_like" op: "Fill" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/ones_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Select" op: "Select" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Greater" input: "bert/encoder/layer_3/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/ones_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Log" op: "Log" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Select" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/zeros_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/zeros_like" op: "Fill" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/zeros_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Select_1" op: "Select" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Greater" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Log" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/zeros_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/mul_2" op: "Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_3/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/mul_3" op: "Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/mul_2" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Select_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/mul_3" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Sum" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_58" op: "AddN" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_3_grad/Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_3/intermediate/dense/Pow_grad/mul_1" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/intermediate/dense/mul_3_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_58" input: "bert/encoder/layer_3/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/add_1" input: "gradients/AddN_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_59" op: "AddN" input: "gradients/AddN_57" input: "gradients/bert/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_59" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_59" input: "bert/encoder/layer_3/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_59" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_60" op: "AddN" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_60" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/Rsqrt" input: "gradients/AddN_60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_3/attention/output/add" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/moments/StopGradient" input: "^gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_61" op: "AddN" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_3/attention/output/dropout/GreaterEqual" input: "gradients/AddN_61" input: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_3/attention/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_3/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/attention/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_3/attention/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_3/attention/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_3/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_3/attention/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_3/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_3/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_3/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_3/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_3/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_3/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_3/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_3/attention/self/dropout/SelectV2" input: "gradients/bert/encoder/layer_3/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_3/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_3/attention/self/MatMul_1_grad/MatMul" input: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_3/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/zeros" input: "gradients/bert/encoder/layer_3/attention/self/MatMul_1_grad/MatMul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_3/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_3/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_3/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_3/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/attention/self/Softmax" input: "gradients/bert/encoder/layer_3/attention/self/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/self/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_3/attention/self/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/self/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/dropout/Mul_grad/Mul" input: "bert/encoder/layer_3/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_3/attention/self/dropout/Mul_grad/Mul" input: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_3/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_3/attention/self/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_3/attention/self/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\001\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/self/add_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_3/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_1" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul_1" input: "bert/encoder/layer_3/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_3/attention/self/MatMul" input: "gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/MatMul_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_3/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_3/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_3/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_3/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_3/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_3/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_3/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_3/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_3/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_3/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_3/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_3/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_1" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_3/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_3/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_1" input: "gradients/bert/encoder/layer_3/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_62" op: "AddN" input: "gradients/AddN_61" input: "gradients/bert/encoder/layer_3/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_3/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_3/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_62" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_62" input: "bert/encoder/layer_2/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_62" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/output/layer_normalization_6/moments/mean" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_63" op: "AddN" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_63" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/Rsqrt" input: "gradients/AddN_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_2/output/layer_normalization_6/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_2/output/add" input: "bert/encoder/layer_2/output/layer_normalization_6/moments/StopGradient" input: "^gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_64" op: "AddN" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_2/output/dropout/GreaterEqual" input: "gradients/AddN_64" input: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_2/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_2/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_2/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_2/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_2/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_2/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_2/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_2/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_2/intermediate/dense/mul_3" input: "gradients/bert/encoder/layer_2/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_3_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_2/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_3_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_2/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_3_grad/Mul_1" input: "bert/encoder/layer_2/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_2_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_2_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/intermediate/dense/mul_2/x" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_3_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/add_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/add_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_2/intermediate/dense/add_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/add_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/add_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/add_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/intermediate/dense/add_1_grad/Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/add_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Tanh_grad/TanhGrad" op: "TanhGrad" input: "bert/encoder/layer_2/intermediate/dense/Tanh" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_2_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/Tanh_grad/TanhGrad" input: "bert/encoder/layer_2/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/intermediate/dense/mul_1/x" input: "gradients/bert/encoder/layer_2/intermediate/dense/Tanh_grad/TanhGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_2/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_2/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/sub/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/sub" op: "Sub" input: "bert/encoder/layer_2/intermediate/dense/Pow/y" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/sub/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Pow" op: "Pow" input: "bert/encoder/layer_2/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Greater/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Greater" op: "Greater" input: "bert/encoder/layer_2/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Greater/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/ones_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/ones_like" op: "Fill" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/ones_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Select" op: "Select" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Greater" input: "bert/encoder/layer_2/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/ones_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Log" op: "Log" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Select" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/zeros_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/zeros_like" op: "Fill" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/zeros_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Select_1" op: "Select" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Greater" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Log" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/zeros_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/mul_2" op: "Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_2/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/mul_3" op: "Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/mul_2" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Select_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/mul_3" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Sum" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_65" op: "AddN" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_3_grad/Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_2/intermediate/dense/Pow_grad/mul_1" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/intermediate/dense/mul_3_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_65" input: "bert/encoder/layer_2/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/add_1" input: "gradients/AddN_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_66" op: "AddN" input: "gradients/AddN_64" input: "gradients/bert/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_66" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_66" input: "bert/encoder/layer_2/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_66" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_67" op: "AddN" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_67" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/Rsqrt" input: "gradients/AddN_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_2/attention/output/add" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/moments/StopGradient" input: "^gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_68" op: "AddN" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_2/attention/output/dropout/GreaterEqual" input: "gradients/AddN_68" input: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_2/attention/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_2/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/attention/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_2/attention/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_2/attention/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_2/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_2/attention/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_2/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_2/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_2/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_2/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_2/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_2/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_2/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_2/attention/self/dropout/SelectV2" input: "gradients/bert/encoder/layer_2/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_2/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_2/attention/self/MatMul_1_grad/MatMul" input: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_2/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/zeros" input: "gradients/bert/encoder/layer_2/attention/self/MatMul_1_grad/MatMul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_2/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_2/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_2/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_2/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/attention/self/Softmax" input: "gradients/bert/encoder/layer_2/attention/self/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/self/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_2/attention/self/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/self/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/dropout/Mul_grad/Mul" input: "bert/encoder/layer_2/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_2/attention/self/dropout/Mul_grad/Mul" input: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_2/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_2/attention/self/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_2/attention/self/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\001\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/self/add_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_2/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_1" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul_1" input: "bert/encoder/layer_2/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_2/attention/self/MatMul" input: "gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/MatMul_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_2/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_2/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_2/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_2/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_2/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_2/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_2/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_2/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_2/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_2/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_2/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_2/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_1" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_2/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_2/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_1" input: "gradients/bert/encoder/layer_2/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_69" op: "AddN" input: "gradients/AddN_68" input: "gradients/bert/encoder/layer_2/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_2/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_2/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_69" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_69" input: "bert/encoder/layer_1/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_69" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/output/layer_normalization_4/moments/mean" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_70" op: "AddN" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_70" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/Rsqrt" input: "gradients/AddN_70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_1/output/layer_normalization_4/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_1/output/add" input: "bert/encoder/layer_1/output/layer_normalization_4/moments/StopGradient" input: "^gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_71" op: "AddN" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_1/output/dropout/GreaterEqual" input: "gradients/AddN_71" input: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_1/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_1/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_1/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_1/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_1/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_1/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_1/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_1/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_1/intermediate/dense/mul_3" input: "gradients/bert/encoder/layer_1/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_3_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_1/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_3_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_1/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_3_grad/Mul_1" input: "bert/encoder/layer_1/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_2_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_2_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/intermediate/dense/mul_2/x" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_3_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/add_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/add_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_1/intermediate/dense/add_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/add_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/add_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/add_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/intermediate/dense/add_1_grad/Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/add_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Tanh_grad/TanhGrad" op: "TanhGrad" input: "bert/encoder/layer_1/intermediate/dense/Tanh" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_2_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/Tanh_grad/TanhGrad" input: "bert/encoder/layer_1/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/intermediate/dense/mul_1/x" input: "gradients/bert/encoder/layer_1/intermediate/dense/Tanh_grad/TanhGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_1/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_1/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/sub/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/sub" op: "Sub" input: "bert/encoder/layer_1/intermediate/dense/Pow/y" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/sub/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Pow" op: "Pow" input: "bert/encoder/layer_1/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Greater/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Greater" op: "Greater" input: "bert/encoder/layer_1/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Greater/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/ones_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/ones_like" op: "Fill" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/ones_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Select" op: "Select" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Greater" input: "bert/encoder/layer_1/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/ones_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Log" op: "Log" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Select" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/zeros_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/zeros_like" op: "Fill" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/zeros_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Select_1" op: "Select" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Greater" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Log" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/zeros_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/mul_2" op: "Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_1/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/mul_3" op: "Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/mul_2" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Select_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/mul_3" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Sum" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_72" op: "AddN" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_3_grad/Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_1/intermediate/dense/Pow_grad/mul_1" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/intermediate/dense/mul_3_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_72" input: "bert/encoder/layer_1/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/add_1" input: "gradients/AddN_72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_73" op: "AddN" input: "gradients/AddN_71" input: "gradients/bert/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_73" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_73" input: "bert/encoder/layer_1/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_73" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_74" op: "AddN" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_74" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/Rsqrt" input: "gradients/AddN_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_1/attention/output/add" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/moments/StopGradient" input: "^gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_75" op: "AddN" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_1/attention/output/dropout/GreaterEqual" input: "gradients/AddN_75" input: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_1/attention/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_1/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/attention/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_1/attention/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_1/attention/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_1/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_1/attention/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_1/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_1/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_1/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_1/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_1/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_1/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_1/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_1/attention/self/dropout/SelectV2" input: "gradients/bert/encoder/layer_1/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_1/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_1/attention/self/MatMul_1_grad/MatMul" input: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_1/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/zeros" input: "gradients/bert/encoder/layer_1/attention/self/MatMul_1_grad/MatMul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_1/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_1/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_1/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_1/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/attention/self/Softmax" input: "gradients/bert/encoder/layer_1/attention/self/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/self/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_1/attention/self/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/self/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/dropout/Mul_grad/Mul" input: "bert/encoder/layer_1/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_1/attention/self/dropout/Mul_grad/Mul" input: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_1/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_1/attention/self/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_1/attention/self/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\001\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/self/add_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_1/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_1" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul_1" input: "bert/encoder/layer_1/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_1/attention/self/MatMul" input: "gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/MatMul_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_1/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_1/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_1/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_1/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_1/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_1/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_1/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_1/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_1/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_1/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_1/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_1/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_1" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_1/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_1/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_1" input: "gradients/bert/encoder/layer_1/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_76" op: "AddN" input: "gradients/AddN_75" input: "gradients/bert/encoder/layer_1/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_1/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_1/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_76" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_76" input: "bert/encoder/layer_0/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_76" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/output/layer_normalization_2/moments/mean" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_77" op: "AddN" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_77" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/Rsqrt" input: "gradients/AddN_77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_0/output/layer_normalization_2/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_0/output/add" input: "bert/encoder/layer_0/output/layer_normalization_2/moments/StopGradient" input: "^gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_78" op: "AddN" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_0/output/dropout/GreaterEqual" input: "gradients/AddN_78" input: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_0/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_0/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_0/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_0/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_0/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_0/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_0/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_0/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_0/intermediate/dense/mul_3" input: "gradients/bert/encoder/layer_0/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_3_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_0/intermediate/dense/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_3_grad/Mul_1" op: "Mul" input: "gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul" input: "bert/encoder/layer_0/intermediate/dense/BiasAdd" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_3_grad/Mul_1" input: "bert/encoder/layer_0/intermediate/dense/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_2_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_2_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/intermediate/dense/mul_2/x" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_3_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/add_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/add_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_2_grad/Mul_1" input: "gradients/bert/encoder/layer_0/intermediate/dense/add_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/add_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/add_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/add_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/intermediate/dense/add_1_grad/Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/add_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Tanh_grad/TanhGrad" op: "TanhGrad" input: "bert/encoder/layer_0/intermediate/dense/Tanh" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_2_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/Tanh_grad/TanhGrad" input: "bert/encoder/layer_0/intermediate/dense/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/intermediate/dense/mul_1/x" input: "gradients/bert/encoder/layer_0/intermediate/dense/Tanh_grad/TanhGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Mul_1" input: "bert/encoder/layer_0/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/intermediate/dense/mul/x" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_0/intermediate/dense/Pow/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/sub/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/sub" op: "Sub" input: "bert/encoder/layer_0/intermediate/dense/Pow/y" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/sub/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Pow" op: "Pow" input: "bert/encoder/layer_0/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Greater/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Greater" op: "Greater" input: "bert/encoder/layer_0/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Greater/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/ones_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/ones_like" op: "Fill" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/ones_like/Shape/shape_as_tensor" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/ones_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Select" op: "Select" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Greater" input: "bert/encoder/layer_0/intermediate/dense/BiasAdd" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/ones_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Log" op: "Log" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Select" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\014\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/zeros_like/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/zeros_like" op: "Fill" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/zeros_like/shape_as_tensor" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/zeros_like/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Select_1" op: "Select" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Greater" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Log" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/zeros_like" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/mul_2" op: "Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_grad/Mul_1" input: "bert/encoder/layer_0/intermediate/dense/Pow" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/mul_3" op: "Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/mul_2" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Select_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/mul_3" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Sum" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/AddN_79" op: "AddN" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_3_grad/Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_0/intermediate/dense/Pow_grad/mul_1" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/intermediate/dense/mul_3_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 3072 } } } } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/AddN_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/AddN_79" input: "bert/encoder/layer_0/intermediate/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/add_1" input: "gradients/AddN_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_80" op: "AddN" input: "gradients/AddN_78" input: "gradients/bert/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/AddN_80" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/AddN_80" input: "bert/encoder/layer_0/attention/output/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/AddN_80" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/AddN_80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub_grad/Neg" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_2_grad/Mul" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_2_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_81" op: "AddN" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_81" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Mul" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/Rsqrt" input: "gradients/AddN_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Mul_1" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Sum_1" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/Rsqrt" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/add_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance_grad/Reshape" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance_grad/Tile" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/scalar" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/encoder/layer_0/attention/output/add" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/moments/StopGradient" input: "^gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/Mul" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean_grad/Reshape" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean_grad/Tile" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/AddN_82" op: "AddN" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_1_grad/Mul" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_0/attention/output/dropout/GreaterEqual" input: "gradients/AddN_82" input: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_0/attention/output/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/zeros" input: "gradients/AddN_82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_0/attention/output/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/attention/output/dense/BiasAdd" input: "gradients/bert/encoder/layer_0/attention/output/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/output/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_0/attention/output/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/output/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_0/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_0/attention/output/dropout/Mul_grad/Mul" input: "bert/encoder/layer_0/attention/output/dense/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/layer_0/attention/self/Reshape_3" input: "gradients/bert/encoder/layer_0/attention/output/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000,\001\000\000\014\000\000\000@\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/transpose_3_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_0/attention/self/transpose_3/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/transpose_3_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_3_grad/Reshape" input: "gradients/bert/encoder/layer_0/attention/self/transpose_3_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/MatMul_1_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_0/attention/self/transpose_3_grad/transpose" input: "bert/encoder/layer_0/attention/self/transpose_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/attention/self/MatMul_1_grad/MatMul_1" op: "BatchMatMulV2" input: "bert/encoder/layer_0/attention/self/dropout/SelectV2" input: "gradients/bert/encoder/layer_0/attention/self/transpose_3_grad/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/encoder/layer_0/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_0/attention/self/MatMul_1_grad/MatMul" input: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\014\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Shape" input: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/encoder/layer_0/attention/self/dropout/GreaterEqual" input: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/zeros" input: "gradients/bert/encoder/layer_0/attention/self/MatMul_1_grad/MatMul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/transpose_2_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_0/attention/self/transpose_2/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/transpose_2_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_0/attention/self/MatMul_1_grad/MatMul_1" input: "gradients/bert/encoder/layer_0/attention/self/transpose_2_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Reshape" input: "bert/encoder/layer_0/attention/self/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/attention/self/Softmax" input: "gradients/bert/encoder/layer_0/attention/self/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/self/dropout/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_0/attention/self/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/dropout/Mul_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/self/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Reshape_2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Reshape_2_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/transpose_2_grad/transpose" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/dropout/Mul_grad/Mul" input: "bert/encoder/layer_0/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: -1 } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul" input: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/sub" op: "Sub" input: "gradients/bert/encoder/layer_0/attention/self/dropout/Mul_grad/Mul" input: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul_1" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/sub" input: "bert/encoder/layer_0/attention/self/Softmax" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_0/attention/self/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/add_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul_1" input: "gradients/bert/encoder/layer_0/attention/self/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\020\000\000\000\001\000\000\000,\001\000\000,\001\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/add_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/add_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/self/add_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 1 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/value/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_2_grad/Reshape" input: "bert/encoder/layer_0/attention/self/value/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/Reshape_1" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Mul" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul_1" input: "bert/encoder/layer_0/attention/self/Mul/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Mul_1" op: "Mul" input: "bert/encoder/layer_0/attention/self/MatMul" input: "gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 300 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 4 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000\003\000\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Sum" op: "Sum" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Mul_1" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Sum" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/MatMul_grad/MatMul" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_0/attention/self/transpose_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: false } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/MatMul_grad/MatMul_1" op: "BatchMatMulV2" input: "gradients/bert/encoder/layer_0/attention/self/Mul_grad/Mul" input: "bert/encoder/layer_0/attention/self/transpose" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 12 } dim { size: 300 } dim { size: 64 } } } } } attr { key: "adj_x" value { b: true } } attr { key: "adj_y" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/transpose_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_0/attention/self/transpose/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/transpose_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_0/attention/self/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_0/attention/self/transpose_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/transpose_1_grad/InvertPermutation" op: "InvertPermutation" input: "bert/encoder/layer_0/attention/self/transpose_1/perm" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/transpose_1_grad/transpose" op: "Transpose" input: "gradients/bert/encoder/layer_0/attention/self/MatMul_grad/MatMul_1" input: "gradients/bert/encoder/layer_0/attention/self/transpose_1_grad/InvertPermutation" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tperm" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 12 } dim { size: 64 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Reshape_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Reshape_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/transpose_grad/transpose" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/bert/encoder/layer_0/attention/self/transpose_1_grad/transpose" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad" op: "BiasAddGrad" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "data_format" value { s: "NHWC" } } } node { name: "gradients/bert/encoder/layer_0/attention/self/query/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_grad/Reshape" input: "bert/encoder/layer_0/attention/self/query/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/Reshape_1" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/encoder/layer_0/attention/self/key/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_1_grad/Reshape" input: "bert/encoder/layer_0/attention/self/key/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/encoder/Reshape_1" input: "gradients/bert/encoder/layer_0/attention/self/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/AddN_83" op: "AddN" input: "gradients/AddN_82" input: "gradients/bert/encoder/layer_0/attention/self/value/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_0/attention/self/query/MatMul_grad/MatMul" input: "gradients/bert/encoder/layer_0/attention/self/key/MatMul_grad/MatMul" attr { key: "N" value { i: 4 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/encoder/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "gradients/bert/encoder/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/AddN_83" input: "gradients/bert/encoder/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/dropout/SelectV2_grad/zeros" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "gradients/bert/embeddings/dropout/SelectV2_grad/SelectV2" op: "SelectV2" input: "bert/embeddings/dropout/GreaterEqual" input: "gradients/bert/encoder/Reshape_1_grad/Reshape" input: "gradients/bert/embeddings/dropout/SelectV2_grad/zeros" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/dropout/SelectV2_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/dropout/SelectV2_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/dropout/SelectV2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/embeddings/dropout/SelectV2_grad/Shape" input: "gradients/bert/embeddings/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/embeddings/dropout/SelectV2_grad/Sum" op: "Sum" input: "gradients/bert/embeddings/dropout/SelectV2_grad/SelectV2" input: "gradients/bert/embeddings/dropout/SelectV2_grad/BroadcastGradientArgs" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/embeddings/dropout/SelectV2_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/dropout/SelectV2_grad/Sum" input: "gradients/bert/embeddings/dropout/SelectV2_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/dropout/SelectV2_grad/SelectV2_1" op: "SelectV2" input: "bert/embeddings/dropout/GreaterEqual" input: "gradients/bert/embeddings/dropout/SelectV2_grad/zeros" input: "gradients/bert/encoder/Reshape_1_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/dropout/SelectV2_grad/Shape_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/embeddings/dropout/SelectV2_grad/BroadcastGradientArgs_1" op: "BroadcastGradientArgs" input: "gradients/bert/embeddings/dropout/SelectV2_grad/Shape_2" input: "gradients/bert/embeddings/dropout/SelectV2_grad/Shape_1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/embeddings/dropout/SelectV2_grad/Sum_1" op: "Sum" input: "gradients/bert/embeddings/dropout/SelectV2_grad/SelectV2_1" input: "gradients/bert/embeddings/dropout/SelectV2_grad/BroadcastGradientArgs_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 1 } dim { size: 1 } } } } } attr { key: "keep_dims" value { b: true } } } node { name: "gradients/bert/embeddings/dropout/SelectV2_grad/Reshape_1" op: "Reshape" input: "gradients/bert/embeddings/dropout/SelectV2_grad/Sum_1" input: "gradients/bert/embeddings/dropout/SelectV2_grad/Shape_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/embeddings/dropout/Mul_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/dropout/Mul_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/embeddings/dropout/Mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/embeddings/dropout/Mul_grad/BroadcastGradientArgs/s0" input: "gradients/bert/embeddings/dropout/Mul_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/embeddings/dropout/Mul_grad/Mul" op: "Mul" input: "gradients/bert/embeddings/dropout/SelectV2_grad/Reshape" input: "bert/embeddings/dropout/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/dropout/Mul_grad/Mul_1" op: "Mul" input: "bert/embeddings/layer_normalization/batchnorm/add_1" input: "gradients/bert/embeddings/dropout/SelectV2_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/dropout/Mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000" } } } } node { name: "gradients/bert/embeddings/dropout/Mul_grad/Sum" op: "Sum" input: "gradients/bert/embeddings/dropout/Mul_grad/Mul_1" input: "gradients/bert/embeddings/dropout/Mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/dropout/Mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/embeddings/dropout/Mul_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/embeddings/dropout/Mul_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/dropout/Mul_grad/Sum" input: "gradients/bert/embeddings/dropout/Mul_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_1_grad/Mul" op: "Mul" input: "gradients/bert/embeddings/dropout/Mul_grad/Mul" input: "bert/embeddings/layer_normalization/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_1_grad/Mul_1" op: "Mul" input: "gradients/bert/embeddings/dropout/Mul_grad/Mul" input: "bert/embeddings/add_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/BroadcastGradientArgs/s0" input: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/Sum" op: "Sum" input: "gradients/bert/embeddings/dropout/Mul_grad/Mul" input: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/Sum" input: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/Neg" op: "Neg" input: "gradients/bert/embeddings/dropout/Mul_grad/Mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\001\000\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/BroadcastGradientArgs/s0" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/Mul" op: "Mul" input: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/Neg" input: "bert/embeddings/layer_normalization/batchnorm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 2 } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/Sum" op: "Sum" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/Mul" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\001\000\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/Sum" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/Mul_1" op: "Mul" input: "bert/embeddings/layer_normalization/moments/mean" input: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/Neg" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/AddN_84" op: "AddN" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_1_grad/Mul_1" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/Mul_1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/layer_normalization/batchnorm/mul_1_grad/Mul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\001\000\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/BroadcastGradientArgs/s0" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Mul" op: "Mul" input: "gradients/AddN_84" input: "bert/embeddings/layer_normalization/batchnorm/mul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 2 } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Sum" op: "Sum" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Mul" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\001\000\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Sum" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Mul_1" op: "Mul" input: "bert/embeddings/layer_normalization/batchnorm/Rsqrt" input: "gradients/AddN_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Sum_1/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\000\000\000\001\000\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Sum_1" op: "Sum" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Mul_1" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Sum_1/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Reshape_1/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 768 } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Reshape_1" op: "Reshape" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Sum_1" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Reshape_1/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/Rsqrt_grad/RsqrtGrad" op: "RsqrtGrad" input: "bert/embeddings/layer_normalization/batchnorm/Rsqrt" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/add_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\001\000\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/add_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/add_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/embeddings/layer_normalization/batchnorm/add_grad/BroadcastGradientArgs/s0" input: "gradients/bert/embeddings/layer_normalization/batchnorm/add_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/add_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\000\000\000\000\001\000\000\000\002\000\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/add_grad/Sum" op: "Sum" input: "gradients/bert/embeddings/layer_normalization/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/embeddings/layer_normalization/batchnorm/add_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/add_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/add_grad/Reshape/shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/batchnorm/add_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/layer_normalization/batchnorm/add_grad/Sum" input: "gradients/bert/embeddings/layer_normalization/batchnorm/add_grad/Reshape/shape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/Maximum/x" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\001\000\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/Maximum/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/Maximum" op: "Maximum" input: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/Maximum/x" input: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/Maximum/y" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/floordiv/x" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/floordiv" op: "FloorDiv" input: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/floordiv/x" input: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/Maximum" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\001\000\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/layer_normalization/batchnorm/Rsqrt_grad/RsqrtGrad" input: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\001\000\000\000\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/Tile" op: "Tile" input: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/Reshape" input: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/truediv" op: "RealDiv" input: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/Tile" input: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/scalar" op: "Const" input: "^gradients/bert/embeddings/layer_normalization/moments/variance_grad/truediv" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/Mul" op: "Mul" input: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/scalar" input: "gradients/bert/embeddings/layer_normalization/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/sub" op: "Sub" input: "bert/embeddings/add_1" input: "bert/embeddings/layer_normalization/moments/StopGradient" input: "^gradients/bert/embeddings/layer_normalization/moments/variance_grad/truediv" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/mul_1" op: "Mul" input: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/Mul" input: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\001\000\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/BroadcastGradientArgs/s0" input: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 2 } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/Sum" op: "Sum" input: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\001\000\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/Sum" input: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/Neg" op: "Neg" input: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/mean_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\001\000\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/mean_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_2_grad/Reshape" input: "gradients/bert/embeddings/layer_normalization/moments/mean_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/mean_grad/Tile/multiples" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\001\000\000\000\001\000\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/mean_grad/Tile" op: "Tile" input: "gradients/bert/embeddings/layer_normalization/moments/mean_grad/Reshape" input: "gradients/bert/embeddings/layer_normalization/moments/mean_grad/Tile/multiples" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tmultiples" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/mean_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 768.0 } } } } node { name: "gradients/bert/embeddings/layer_normalization/moments/mean_grad/truediv" op: "RealDiv" input: "gradients/bert/embeddings/layer_normalization/moments/mean_grad/Tile" input: "gradients/bert/embeddings/layer_normalization/moments/mean_grad/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/AddN_85" op: "AddN" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_1_grad/Mul" input: "gradients/bert/embeddings/layer_normalization/moments/SquaredDifference_grad/mul_1" input: "gradients/bert/embeddings/layer_normalization/moments/mean_grad/truediv" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/layer_normalization/batchnorm/mul_1_grad/Mul" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 16 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/add_1_grad/BroadcastGradientArgs/s0" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\020\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/add_1_grad/BroadcastGradientArgs/s1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\001\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/add_1_grad/BroadcastGradientArgs" op: "BroadcastGradientArgs" input: "gradients/bert/embeddings/add_1_grad/BroadcastGradientArgs/s0" input: "gradients/bert/embeddings/add_1_grad/BroadcastGradientArgs/s1" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } shape { dim { size: -1 } } } } } } node { name: "gradients/bert/embeddings/add_1_grad/Sum/reduction_indices" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/embeddings/add_1_grad/Sum" op: "Sum" input: "gradients/AddN_85" input: "gradients/bert/embeddings/add_1_grad/Sum/reduction_indices" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 300 } dim { size: 768 } } } } } attr { key: "keep_dims" value { b: false } } } node { name: "gradients/bert/embeddings/add_1_grad/Reshape/shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 3 } } tensor_content: "\001\000\000\000,\001\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/add_1_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/add_1_grad/Sum" input: "gradients/bert/embeddings/add_1_grad/Reshape/shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/Reshape_4_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: ",\001\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/Reshape_4_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/add_1_grad/Reshape" input: "gradients/bert/embeddings/Reshape_4_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 300 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/Reshape_1_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/Reshape_1_grad/Reshape" op: "Reshape" input: "gradients/AddN_85" input: "gradients/bert/embeddings/Reshape_1_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/Reshape_3_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\300\022\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/Reshape_3_grad/Reshape" op: "Reshape" input: "gradients/AddN_85" input: "gradients/bert/embeddings/Reshape_3_grad/Shape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/Slice_grad/Rank" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 2 } } } } node { name: "gradients/bert/embeddings/Slice_grad/Shape" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: ",\001\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/Slice_grad/stack/1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/embeddings/Slice_grad/stack" op: "Pack" input: "gradients/bert/embeddings/Slice_grad/Rank" input: "gradients/bert/embeddings/Slice_grad/stack/1" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "axis" value { i: 0 } } } node { name: "gradients/bert/embeddings/Slice_grad/Reshape" op: "Reshape" input: "bert/embeddings/Slice/begin" input: "gradients/bert/embeddings/Slice_grad/stack" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/Slice_grad/Shape_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\002\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/Slice_grad/sub" op: "Sub" input: "gradients/bert/embeddings/Slice_grad/Shape_1" input: "gradients/bert/embeddings/Slice_grad/Shape" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/embeddings/Slice_grad/sub_1" op: "Sub" input: "gradients/bert/embeddings/Slice_grad/sub" input: "bert/embeddings/Slice/begin" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/embeddings/Slice_grad/Reshape_1" op: "Reshape" input: "gradients/bert/embeddings/Slice_grad/sub_1" input: "gradients/bert/embeddings/Slice_grad/stack" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/Slice_grad/concat/axis" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "gradients/bert/embeddings/Slice_grad/concat" op: "ConcatV2" input: "gradients/bert/embeddings/Slice_grad/Reshape" input: "gradients/bert/embeddings/Slice_grad/Reshape_1" input: "gradients/bert/embeddings/Slice_grad/concat/axis" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 2 } } } } } } node { name: "gradients/bert/embeddings/Slice_grad/Pad" op: "Pad" input: "gradients/bert/embeddings/Reshape_4_grad/Reshape" input: "gradients/bert/embeddings/Slice_grad/concat" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tpaddings" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/MatMul_grad/MatMul" op: "MatMul" input: "gradients/bert/embeddings/Reshape_3_grad/Reshape" input: "bert/embeddings/MatMul/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 2 } } } } } attr { key: "transpose_a" value { b: false } } attr { key: "transpose_b" value { b: true } } } node { name: "gradients/bert/embeddings/MatMul_grad/MatMul_1" op: "MatMul" input: "bert/embeddings/one_hot" input: "gradients/bert/embeddings/Reshape_3_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "transpose_a" value { b: true } } attr { key: "transpose_b" value { b: false } } } node { name: "gradients/bert/embeddings/Gather_grad/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\210R\000\000\000\003\000\000" } } } } node { name: "gradients/bert/embeddings/Gather_grad/Size" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 4800 } } } } node { name: "gradients/bert/embeddings/Gather_grad/ExpandDims/dim" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/embeddings/Gather_grad/ExpandDims" op: "ExpandDims" input: "gradients/bert/embeddings/Gather_grad/Size" input: "gradients/bert/embeddings/Gather_grad/ExpandDims/dim" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tdim" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "gradients/bert/embeddings/Gather_grad/strided_slice/stack" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/embeddings/Gather_grad/strided_slice/stack_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "gradients/bert/embeddings/Gather_grad/strided_slice/stack_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "gradients/bert/embeddings/Gather_grad/strided_slice" op: "StridedSlice" input: "gradients/bert/embeddings/Gather_grad/Const" input: "gradients/bert/embeddings/Gather_grad/strided_slice/stack" input: "gradients/bert/embeddings/Gather_grad/strided_slice/stack_1" input: "gradients/bert/embeddings/Gather_grad/strided_slice/stack_2" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 1 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "gradients/bert/embeddings/Gather_grad/concat/axis" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "gradients/bert/embeddings/Gather_grad/concat" op: "ConcatV2" input: "gradients/bert/embeddings/Gather_grad/ExpandDims" input: "gradients/bert/embeddings/Gather_grad/strided_slice" input: "gradients/bert/embeddings/Gather_grad/concat/axis" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } } node { name: "gradients/bert/embeddings/Gather_grad/Reshape" op: "Reshape" input: "gradients/bert/embeddings/Reshape_1_grad/Reshape" input: "gradients/bert/embeddings/Gather_grad/concat" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "gradients/bert/embeddings/Gather_grad/Reshape_1" op: "Reshape" input: "bert/embeddings/Reshape" input: "gradients/bert/embeddings/Gather_grad/ExpandDims" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } } } } } } node { name: "global_norm/L2Loss" op: "L2Loss" input: "gradients/bert/embeddings/Gather_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/Gather_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_1" op: "L2Loss" input: "gradients/bert/embeddings/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_2" op: "L2Loss" input: "gradients/bert/embeddings/Slice_grad/Pad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/Slice_grad/Pad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_3" op: "L2Loss" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_4" op: "L2Loss" input: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_5" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_6" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_7" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_8" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_9" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_10" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_11" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_12" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_13" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_14" op: "L2Loss" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_15" op: "L2Loss" input: "gradients/bert/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_16" op: "L2Loss" input: "gradients/bert/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_17" op: "L2Loss" input: "gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_18" op: "L2Loss" input: "gradients/bert/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_19" op: "L2Loss" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_20" op: "L2Loss" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_21" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_22" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_23" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_24" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_25" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_26" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_27" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_28" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_29" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_30" op: "L2Loss" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_31" op: "L2Loss" input: "gradients/bert/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_32" op: "L2Loss" input: "gradients/bert/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_33" op: "L2Loss" input: "gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_34" op: "L2Loss" input: "gradients/bert/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_35" op: "L2Loss" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_36" op: "L2Loss" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_37" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_38" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_39" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_40" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_41" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_42" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_43" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_44" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_45" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_46" op: "L2Loss" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_47" op: "L2Loss" input: "gradients/bert/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_48" op: "L2Loss" input: "gradients/bert/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_49" op: "L2Loss" input: "gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_50" op: "L2Loss" input: "gradients/bert/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_51" op: "L2Loss" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_52" op: "L2Loss" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_53" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_54" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_55" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_56" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_57" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_58" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_59" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_60" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_61" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_62" op: "L2Loss" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_63" op: "L2Loss" input: "gradients/bert/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_64" op: "L2Loss" input: "gradients/bert/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_65" op: "L2Loss" input: "gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_66" op: "L2Loss" input: "gradients/bert/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_67" op: "L2Loss" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_68" op: "L2Loss" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_69" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_70" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_71" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_72" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_73" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_74" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_75" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_76" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_77" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_78" op: "L2Loss" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_79" op: "L2Loss" input: "gradients/bert/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_80" op: "L2Loss" input: "gradients/bert/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_81" op: "L2Loss" input: "gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_82" op: "L2Loss" input: "gradients/bert/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_83" op: "L2Loss" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_84" op: "L2Loss" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_85" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_86" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_87" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_88" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_89" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_90" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_91" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_92" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_93" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_94" op: "L2Loss" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_95" op: "L2Loss" input: "gradients/bert/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_96" op: "L2Loss" input: "gradients/bert/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_97" op: "L2Loss" input: "gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_98" op: "L2Loss" input: "gradients/bert/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_99" op: "L2Loss" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_100" op: "L2Loss" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_101" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_102" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_103" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_104" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_105" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_106" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_107" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_108" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_109" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_110" op: "L2Loss" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_111" op: "L2Loss" input: "gradients/bert/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_112" op: "L2Loss" input: "gradients/bert/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_113" op: "L2Loss" input: "gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_114" op: "L2Loss" input: "gradients/bert/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_115" op: "L2Loss" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_116" op: "L2Loss" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_117" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_118" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_119" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_120" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_121" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_122" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_123" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_124" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_125" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_126" op: "L2Loss" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_127" op: "L2Loss" input: "gradients/bert/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_128" op: "L2Loss" input: "gradients/bert/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_129" op: "L2Loss" input: "gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_130" op: "L2Loss" input: "gradients/bert/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_131" op: "L2Loss" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_132" op: "L2Loss" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_133" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_134" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_135" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_136" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_137" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_138" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_139" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_140" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_141" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_142" op: "L2Loss" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_143" op: "L2Loss" input: "gradients/bert/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_144" op: "L2Loss" input: "gradients/bert/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_145" op: "L2Loss" input: "gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_146" op: "L2Loss" input: "gradients/bert/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_147" op: "L2Loss" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_148" op: "L2Loss" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_149" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_150" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_151" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_152" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_153" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_154" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_155" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_156" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_157" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_158" op: "L2Loss" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_159" op: "L2Loss" input: "gradients/bert/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_160" op: "L2Loss" input: "gradients/bert/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_161" op: "L2Loss" input: "gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_162" op: "L2Loss" input: "gradients/bert/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_163" op: "L2Loss" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_164" op: "L2Loss" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_165" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_166" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_167" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_168" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_169" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_170" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_171" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_172" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_173" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_174" op: "L2Loss" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_175" op: "L2Loss" input: "gradients/bert/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_176" op: "L2Loss" input: "gradients/bert/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_177" op: "L2Loss" input: "gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_178" op: "L2Loss" input: "gradients/bert/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_179" op: "L2Loss" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_180" op: "L2Loss" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_181" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_182" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_183" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_184" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_185" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_186" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_187" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_188" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_189" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_190" op: "L2Loss" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_191" op: "L2Loss" input: "gradients/bert/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_192" op: "L2Loss" input: "gradients/bert/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_193" op: "L2Loss" input: "gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_194" op: "L2Loss" input: "gradients/bert/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_195" op: "L2Loss" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Reshape_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_196" op: "L2Loss" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/Reshape" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_197" op: "L2Loss" input: "gradients/bert/pooler/dense/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/pooler/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_198" op: "L2Loss" input: "gradients/bert/pooler/dense/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/pooler/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_199" op: "L2Loss" input: "gradients/loss/MatMul_grad/MatMul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/loss/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/L2Loss_200" op: "L2Loss" input: "gradients/loss/BiasAdd_grad/BiasAddGrad" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/loss/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/stack" op: "Pack" input: "global_norm/L2Loss" input: "global_norm/L2Loss_1" input: "global_norm/L2Loss_2" input: "global_norm/L2Loss_3" input: "global_norm/L2Loss_4" input: "global_norm/L2Loss_5" input: "global_norm/L2Loss_6" input: "global_norm/L2Loss_7" input: "global_norm/L2Loss_8" input: "global_norm/L2Loss_9" input: "global_norm/L2Loss_10" input: "global_norm/L2Loss_11" input: "global_norm/L2Loss_12" input: "global_norm/L2Loss_13" input: "global_norm/L2Loss_14" input: "global_norm/L2Loss_15" input: "global_norm/L2Loss_16" input: "global_norm/L2Loss_17" input: "global_norm/L2Loss_18" input: "global_norm/L2Loss_19" input: "global_norm/L2Loss_20" input: "global_norm/L2Loss_21" input: "global_norm/L2Loss_22" input: "global_norm/L2Loss_23" input: "global_norm/L2Loss_24" input: "global_norm/L2Loss_25" input: "global_norm/L2Loss_26" input: "global_norm/L2Loss_27" input: "global_norm/L2Loss_28" input: "global_norm/L2Loss_29" input: "global_norm/L2Loss_30" input: "global_norm/L2Loss_31" input: "global_norm/L2Loss_32" input: "global_norm/L2Loss_33" input: "global_norm/L2Loss_34" input: "global_norm/L2Loss_35" input: "global_norm/L2Loss_36" input: "global_norm/L2Loss_37" input: "global_norm/L2Loss_38" input: "global_norm/L2Loss_39" input: "global_norm/L2Loss_40" input: "global_norm/L2Loss_41" input: "global_norm/L2Loss_42" input: "global_norm/L2Loss_43" input: "global_norm/L2Loss_44" input: "global_norm/L2Loss_45" input: "global_norm/L2Loss_46" input: "global_norm/L2Loss_47" input: "global_norm/L2Loss_48" input: "global_norm/L2Loss_49" input: "global_norm/L2Loss_50" input: "global_norm/L2Loss_51" input: "global_norm/L2Loss_52" input: "global_norm/L2Loss_53" input: "global_norm/L2Loss_54" input: "global_norm/L2Loss_55" input: "global_norm/L2Loss_56" input: "global_norm/L2Loss_57" input: "global_norm/L2Loss_58" input: "global_norm/L2Loss_59" input: "global_norm/L2Loss_60" input: "global_norm/L2Loss_61" input: "global_norm/L2Loss_62" input: "global_norm/L2Loss_63" input: "global_norm/L2Loss_64" input: "global_norm/L2Loss_65" input: "global_norm/L2Loss_66" input: "global_norm/L2Loss_67" input: "global_norm/L2Loss_68" input: "global_norm/L2Loss_69" input: "global_norm/L2Loss_70" input: "global_norm/L2Loss_71" input: "global_norm/L2Loss_72" input: "global_norm/L2Loss_73" input: "global_norm/L2Loss_74" input: "global_norm/L2Loss_75" input: "global_norm/L2Loss_76" input: "global_norm/L2Loss_77" input: "global_norm/L2Loss_78" input: "global_norm/L2Loss_79" input: "global_norm/L2Loss_80" input: "global_norm/L2Loss_81" input: "global_norm/L2Loss_82" input: "global_norm/L2Loss_83" input: "global_norm/L2Loss_84" input: "global_norm/L2Loss_85" input: "global_norm/L2Loss_86" input: "global_norm/L2Loss_87" input: "global_norm/L2Loss_88" input: "global_norm/L2Loss_89" input: "global_norm/L2Loss_90" input: "global_norm/L2Loss_91" input: "global_norm/L2Loss_92" input: "global_norm/L2Loss_93" input: "global_norm/L2Loss_94" input: "global_norm/L2Loss_95" input: "global_norm/L2Loss_96" input: "global_norm/L2Loss_97" input: "global_norm/L2Loss_98" input: "global_norm/L2Loss_99" input: "global_norm/L2Loss_100" input: "global_norm/L2Loss_101" input: "global_norm/L2Loss_102" input: "global_norm/L2Loss_103" input: "global_norm/L2Loss_104" input: "global_norm/L2Loss_105" input: "global_norm/L2Loss_106" input: "global_norm/L2Loss_107" input: "global_norm/L2Loss_108" input: "global_norm/L2Loss_109" input: "global_norm/L2Loss_110" input: "global_norm/L2Loss_111" input: "global_norm/L2Loss_112" input: "global_norm/L2Loss_113" input: "global_norm/L2Loss_114" input: "global_norm/L2Loss_115" input: "global_norm/L2Loss_116" input: "global_norm/L2Loss_117" input: "global_norm/L2Loss_118" input: "global_norm/L2Loss_119" input: "global_norm/L2Loss_120" input: "global_norm/L2Loss_121" input: "global_norm/L2Loss_122" input: "global_norm/L2Loss_123" input: "global_norm/L2Loss_124" input: "global_norm/L2Loss_125" input: "global_norm/L2Loss_126" input: "global_norm/L2Loss_127" input: "global_norm/L2Loss_128" input: "global_norm/L2Loss_129" input: "global_norm/L2Loss_130" input: "global_norm/L2Loss_131" input: "global_norm/L2Loss_132" input: "global_norm/L2Loss_133" input: "global_norm/L2Loss_134" input: "global_norm/L2Loss_135" input: "global_norm/L2Loss_136" input: "global_norm/L2Loss_137" input: "global_norm/L2Loss_138" input: "global_norm/L2Loss_139" input: "global_norm/L2Loss_140" input: "global_norm/L2Loss_141" input: "global_norm/L2Loss_142" input: "global_norm/L2Loss_143" input: "global_norm/L2Loss_144" input: "global_norm/L2Loss_145" input: "global_norm/L2Loss_146" input: "global_norm/L2Loss_147" input: "global_norm/L2Loss_148" input: "global_norm/L2Loss_149" input: "global_norm/L2Loss_150" input: "global_norm/L2Loss_151" input: "global_norm/L2Loss_152" input: "global_norm/L2Loss_153" input: "global_norm/L2Loss_154" input: "global_norm/L2Loss_155" input: "global_norm/L2Loss_156" input: "global_norm/L2Loss_157" input: "global_norm/L2Loss_158" input: "global_norm/L2Loss_159" input: "global_norm/L2Loss_160" input: "global_norm/L2Loss_161" input: "global_norm/L2Loss_162" input: "global_norm/L2Loss_163" input: "global_norm/L2Loss_164" input: "global_norm/L2Loss_165" input: "global_norm/L2Loss_166" input: "global_norm/L2Loss_167" input: "global_norm/L2Loss_168" input: "global_norm/L2Loss_169" input: "global_norm/L2Loss_170" input: "global_norm/L2Loss_171" input: "global_norm/L2Loss_172" input: "global_norm/L2Loss_173" input: "global_norm/L2Loss_174" input: "global_norm/L2Loss_175" input: "global_norm/L2Loss_176" input: "global_norm/L2Loss_177" input: "global_norm/L2Loss_178" input: "global_norm/L2Loss_179" input: "global_norm/L2Loss_180" input: "global_norm/L2Loss_181" input: "global_norm/L2Loss_182" input: "global_norm/L2Loss_183" input: "global_norm/L2Loss_184" input: "global_norm/L2Loss_185" input: "global_norm/L2Loss_186" input: "global_norm/L2Loss_187" input: "global_norm/L2Loss_188" input: "global_norm/L2Loss_189" input: "global_norm/L2Loss_190" input: "global_norm/L2Loss_191" input: "global_norm/L2Loss_192" input: "global_norm/L2Loss_193" input: "global_norm/L2Loss_194" input: "global_norm/L2Loss_195" input: "global_norm/L2Loss_196" input: "global_norm/L2Loss_197" input: "global_norm/L2Loss_198" input: "global_norm/L2Loss_199" input: "global_norm/L2Loss_200" attr { key: "N" value { i: 201 } } attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 201 } } } } } attr { key: "axis" value { i: 0 } } } node { name: "global_norm/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "global_norm/Sum" op: "Sum" input: "global_norm/stack" input: "global_norm/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "global_norm/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 2.0 } } } } node { name: "global_norm/mul" op: "Mul" input: "global_norm/Sum" input: "global_norm/Const_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "global_norm/global_norm" op: "Sqrt" input: "global_norm/mul" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "clip_by_global_norm/truediv/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "clip_by_global_norm/truediv" op: "RealDiv" input: "clip_by_global_norm/truediv/x" input: "global_norm/global_norm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "clip_by_global_norm/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "clip_by_global_norm/truediv_1/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "clip_by_global_norm/truediv_1" op: "RealDiv" input: "clip_by_global_norm/Const" input: "clip_by_global_norm/truediv_1/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "clip_by_global_norm/Minimum" op: "Minimum" input: "clip_by_global_norm/truediv" input: "clip_by_global_norm/truediv_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "clip_by_global_norm/mul/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 1.0 } } } } node { name: "clip_by_global_norm/mul" op: "Mul" input: "clip_by_global_norm/mul/x" input: "clip_by_global_norm/Minimum" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "clip_by_global_norm/sub" op: "Sub" input: "global_norm/global_norm" input: "global_norm/global_norm" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "clip_by_global_norm/add" op: "AddV2" input: "clip_by_global_norm/mul" input: "clip_by_global_norm/sub" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "clip_by_global_norm/mul_1" op: "Mul" input: "gradients/bert/embeddings/Gather_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/Gather_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_0" op: "Identity" input: "clip_by_global_norm/mul_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/Gather_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 4800 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_2" op: "Mul" input: "gradients/bert/embeddings/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_1" op: "Identity" input: "clip_by_global_norm/mul_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_3" op: "Mul" input: "gradients/bert/embeddings/Slice_grad/Pad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/Slice_grad/Pad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_2" op: "Identity" input: "clip_by_global_norm/mul_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/Slice_grad/Pad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_4" op: "Mul" input: "gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_3" op: "Identity" input: "clip_by_global_norm/mul_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/layer_normalization/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_5" op: "Mul" input: "gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_4" op: "Identity" input: "clip_by_global_norm/mul_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/embeddings/layer_normalization/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_6" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_5" op: "Identity" input: "clip_by_global_norm/mul_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_7" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_6" op: "Identity" input: "clip_by_global_norm/mul_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_8" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_7" op: "Identity" input: "clip_by_global_norm/mul_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_9" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_8" op: "Identity" input: "clip_by_global_norm/mul_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_10" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_9" op: "Identity" input: "clip_by_global_norm/mul_10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_11" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_10" op: "Identity" input: "clip_by_global_norm/mul_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_12" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_11" op: "Identity" input: "clip_by_global_norm/mul_12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_13" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_12" op: "Identity" input: "clip_by_global_norm/mul_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_14" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_13" op: "Identity" input: "clip_by_global_norm/mul_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_15" op: "Mul" input: "gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_14" op: "Identity" input: "clip_by_global_norm/mul_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/attention/output/layer_normalization_1/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_16" op: "Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_15" op: "Identity" input: "clip_by_global_norm/mul_16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_17" op: "Mul" input: "gradients/bert/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_16" op: "Identity" input: "clip_by_global_norm/mul_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_18" op: "Mul" input: "gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_17" op: "Identity" input: "clip_by_global_norm/mul_18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_19" op: "Mul" input: "gradients/bert/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_18" op: "Identity" input: "clip_by_global_norm/mul_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_20" op: "Mul" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_19" op: "Identity" input: "clip_by_global_norm/mul_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_21" op: "Mul" input: "gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_20" op: "Identity" input: "clip_by_global_norm/mul_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_0/output/layer_normalization_2/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_22" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_21" op: "Identity" input: "clip_by_global_norm/mul_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_23" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_22" op: "Identity" input: "clip_by_global_norm/mul_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_24" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_23" op: "Identity" input: "clip_by_global_norm/mul_24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_25" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_24" op: "Identity" input: "clip_by_global_norm/mul_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_26" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_25" op: "Identity" input: "clip_by_global_norm/mul_26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_27" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_26" op: "Identity" input: "clip_by_global_norm/mul_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_28" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_27" op: "Identity" input: "clip_by_global_norm/mul_28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_29" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_28" op: "Identity" input: "clip_by_global_norm/mul_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_30" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_29" op: "Identity" input: "clip_by_global_norm/mul_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_31" op: "Mul" input: "gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_30" op: "Identity" input: "clip_by_global_norm/mul_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/attention/output/layer_normalization_3/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_32" op: "Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_31" op: "Identity" input: "clip_by_global_norm/mul_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_33" op: "Mul" input: "gradients/bert/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_32" op: "Identity" input: "clip_by_global_norm/mul_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_34" op: "Mul" input: "gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_33" op: "Identity" input: "clip_by_global_norm/mul_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_35" op: "Mul" input: "gradients/bert/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_34" op: "Identity" input: "clip_by_global_norm/mul_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_36" op: "Mul" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_35" op: "Identity" input: "clip_by_global_norm/mul_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_37" op: "Mul" input: "gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_36" op: "Identity" input: "clip_by_global_norm/mul_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_1/output/layer_normalization_4/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_38" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_37" op: "Identity" input: "clip_by_global_norm/mul_38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_39" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_38" op: "Identity" input: "clip_by_global_norm/mul_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_40" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_39" op: "Identity" input: "clip_by_global_norm/mul_40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_41" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_40" op: "Identity" input: "clip_by_global_norm/mul_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_42" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_41" op: "Identity" input: "clip_by_global_norm/mul_42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_43" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_42" op: "Identity" input: "clip_by_global_norm/mul_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_44" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_43" op: "Identity" input: "clip_by_global_norm/mul_44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_45" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_44" op: "Identity" input: "clip_by_global_norm/mul_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_46" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_45" op: "Identity" input: "clip_by_global_norm/mul_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_47" op: "Mul" input: "gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_46" op: "Identity" input: "clip_by_global_norm/mul_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/attention/output/layer_normalization_5/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_48" op: "Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_47" op: "Identity" input: "clip_by_global_norm/mul_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_49" op: "Mul" input: "gradients/bert/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_48" op: "Identity" input: "clip_by_global_norm/mul_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_50" op: "Mul" input: "gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_49" op: "Identity" input: "clip_by_global_norm/mul_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_51" op: "Mul" input: "gradients/bert/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_50" op: "Identity" input: "clip_by_global_norm/mul_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_52" op: "Mul" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_51" op: "Identity" input: "clip_by_global_norm/mul_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_53" op: "Mul" input: "gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_52" op: "Identity" input: "clip_by_global_norm/mul_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_2/output/layer_normalization_6/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_54" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_53" op: "Identity" input: "clip_by_global_norm/mul_54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_55" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_54" op: "Identity" input: "clip_by_global_norm/mul_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_56" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_55" op: "Identity" input: "clip_by_global_norm/mul_56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_57" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_56" op: "Identity" input: "clip_by_global_norm/mul_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_58" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_57" op: "Identity" input: "clip_by_global_norm/mul_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_59" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_58" op: "Identity" input: "clip_by_global_norm/mul_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_60" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_59" op: "Identity" input: "clip_by_global_norm/mul_60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_61" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_60" op: "Identity" input: "clip_by_global_norm/mul_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_62" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_61" op: "Identity" input: "clip_by_global_norm/mul_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_63" op: "Mul" input: "gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_62" op: "Identity" input: "clip_by_global_norm/mul_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/attention/output/layer_normalization_7/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_64" op: "Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_63" op: "Identity" input: "clip_by_global_norm/mul_64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_65" op: "Mul" input: "gradients/bert/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_64" op: "Identity" input: "clip_by_global_norm/mul_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_66" op: "Mul" input: "gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_65" op: "Identity" input: "clip_by_global_norm/mul_66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_67" op: "Mul" input: "gradients/bert/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_66" op: "Identity" input: "clip_by_global_norm/mul_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_68" op: "Mul" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_67" op: "Identity" input: "clip_by_global_norm/mul_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_69" op: "Mul" input: "gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_68" op: "Identity" input: "clip_by_global_norm/mul_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_3/output/layer_normalization_8/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_70" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_69" op: "Identity" input: "clip_by_global_norm/mul_70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_71" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_70" op: "Identity" input: "clip_by_global_norm/mul_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_72" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_71" op: "Identity" input: "clip_by_global_norm/mul_72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_73" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_72" op: "Identity" input: "clip_by_global_norm/mul_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_74" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_73" op: "Identity" input: "clip_by_global_norm/mul_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_75" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_74" op: "Identity" input: "clip_by_global_norm/mul_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_76" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_75" op: "Identity" input: "clip_by_global_norm/mul_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_77" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_76" op: "Identity" input: "clip_by_global_norm/mul_77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_78" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_77" op: "Identity" input: "clip_by_global_norm/mul_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_79" op: "Mul" input: "gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_78" op: "Identity" input: "clip_by_global_norm/mul_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/attention/output/layer_normalization_9/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_80" op: "Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_79" op: "Identity" input: "clip_by_global_norm/mul_80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_81" op: "Mul" input: "gradients/bert/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_80" op: "Identity" input: "clip_by_global_norm/mul_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_82" op: "Mul" input: "gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_81" op: "Identity" input: "clip_by_global_norm/mul_82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_83" op: "Mul" input: "gradients/bert/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_82" op: "Identity" input: "clip_by_global_norm/mul_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_84" op: "Mul" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_83" op: "Identity" input: "clip_by_global_norm/mul_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_85" op: "Mul" input: "gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_84" op: "Identity" input: "clip_by_global_norm/mul_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_4/output/layer_normalization_10/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_86" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_85" op: "Identity" input: "clip_by_global_norm/mul_86" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_87" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_86" op: "Identity" input: "clip_by_global_norm/mul_87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_88" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_87" op: "Identity" input: "clip_by_global_norm/mul_88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_89" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_88" op: "Identity" input: "clip_by_global_norm/mul_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_90" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_89" op: "Identity" input: "clip_by_global_norm/mul_90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_91" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_90" op: "Identity" input: "clip_by_global_norm/mul_91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_92" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_91" op: "Identity" input: "clip_by_global_norm/mul_92" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_93" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_92" op: "Identity" input: "clip_by_global_norm/mul_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_94" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_93" op: "Identity" input: "clip_by_global_norm/mul_94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_95" op: "Mul" input: "gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_94" op: "Identity" input: "clip_by_global_norm/mul_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/attention/output/layer_normalization_11/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_96" op: "Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_95" op: "Identity" input: "clip_by_global_norm/mul_96" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_97" op: "Mul" input: "gradients/bert/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_96" op: "Identity" input: "clip_by_global_norm/mul_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_98" op: "Mul" input: "gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_97" op: "Identity" input: "clip_by_global_norm/mul_98" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_99" op: "Mul" input: "gradients/bert/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_98" op: "Identity" input: "clip_by_global_norm/mul_99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_100" op: "Mul" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_99" op: "Identity" input: "clip_by_global_norm/mul_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_101" op: "Mul" input: "gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_100" op: "Identity" input: "clip_by_global_norm/mul_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_5/output/layer_normalization_12/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_102" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_101" op: "Identity" input: "clip_by_global_norm/mul_102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_103" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_102" op: "Identity" input: "clip_by_global_norm/mul_103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_104" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_103" op: "Identity" input: "clip_by_global_norm/mul_104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_105" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_104" op: "Identity" input: "clip_by_global_norm/mul_105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_106" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_105" op: "Identity" input: "clip_by_global_norm/mul_106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_107" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_106" op: "Identity" input: "clip_by_global_norm/mul_107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_108" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_107" op: "Identity" input: "clip_by_global_norm/mul_108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_109" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_108" op: "Identity" input: "clip_by_global_norm/mul_109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_110" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_109" op: "Identity" input: "clip_by_global_norm/mul_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_111" op: "Mul" input: "gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_110" op: "Identity" input: "clip_by_global_norm/mul_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/attention/output/layer_normalization_13/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_112" op: "Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_111" op: "Identity" input: "clip_by_global_norm/mul_112" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_113" op: "Mul" input: "gradients/bert/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_112" op: "Identity" input: "clip_by_global_norm/mul_113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_114" op: "Mul" input: "gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_113" op: "Identity" input: "clip_by_global_norm/mul_114" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_115" op: "Mul" input: "gradients/bert/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_114" op: "Identity" input: "clip_by_global_norm/mul_115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_116" op: "Mul" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_115" op: "Identity" input: "clip_by_global_norm/mul_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_117" op: "Mul" input: "gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_116" op: "Identity" input: "clip_by_global_norm/mul_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_6/output/layer_normalization_14/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_118" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_117" op: "Identity" input: "clip_by_global_norm/mul_118" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_119" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_118" op: "Identity" input: "clip_by_global_norm/mul_119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_120" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_119" op: "Identity" input: "clip_by_global_norm/mul_120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_121" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_120" op: "Identity" input: "clip_by_global_norm/mul_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_122" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_121" op: "Identity" input: "clip_by_global_norm/mul_122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_123" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_122" op: "Identity" input: "clip_by_global_norm/mul_123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_124" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_123" op: "Identity" input: "clip_by_global_norm/mul_124" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_125" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_124" op: "Identity" input: "clip_by_global_norm/mul_125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_126" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_125" op: "Identity" input: "clip_by_global_norm/mul_126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_127" op: "Mul" input: "gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_126" op: "Identity" input: "clip_by_global_norm/mul_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/attention/output/layer_normalization_15/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_128" op: "Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_127" op: "Identity" input: "clip_by_global_norm/mul_128" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_129" op: "Mul" input: "gradients/bert/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_128" op: "Identity" input: "clip_by_global_norm/mul_129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_130" op: "Mul" input: "gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_129" op: "Identity" input: "clip_by_global_norm/mul_130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_131" op: "Mul" input: "gradients/bert/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_130" op: "Identity" input: "clip_by_global_norm/mul_131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_132" op: "Mul" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_131" op: "Identity" input: "clip_by_global_norm/mul_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_133" op: "Mul" input: "gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_132" op: "Identity" input: "clip_by_global_norm/mul_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_7/output/layer_normalization_16/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_134" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_133" op: "Identity" input: "clip_by_global_norm/mul_134" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_135" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_134" op: "Identity" input: "clip_by_global_norm/mul_135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_136" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_135" op: "Identity" input: "clip_by_global_norm/mul_136" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_137" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_136" op: "Identity" input: "clip_by_global_norm/mul_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_138" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_137" op: "Identity" input: "clip_by_global_norm/mul_138" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_139" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_138" op: "Identity" input: "clip_by_global_norm/mul_139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_140" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_139" op: "Identity" input: "clip_by_global_norm/mul_140" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_141" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_140" op: "Identity" input: "clip_by_global_norm/mul_141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_142" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_141" op: "Identity" input: "clip_by_global_norm/mul_142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_143" op: "Mul" input: "gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_142" op: "Identity" input: "clip_by_global_norm/mul_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/attention/output/layer_normalization_17/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_144" op: "Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_143" op: "Identity" input: "clip_by_global_norm/mul_144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_145" op: "Mul" input: "gradients/bert/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_144" op: "Identity" input: "clip_by_global_norm/mul_145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_146" op: "Mul" input: "gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_145" op: "Identity" input: "clip_by_global_norm/mul_146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_147" op: "Mul" input: "gradients/bert/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_146" op: "Identity" input: "clip_by_global_norm/mul_147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_148" op: "Mul" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_147" op: "Identity" input: "clip_by_global_norm/mul_148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_149" op: "Mul" input: "gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_148" op: "Identity" input: "clip_by_global_norm/mul_149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_8/output/layer_normalization_18/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_150" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_149" op: "Identity" input: "clip_by_global_norm/mul_150" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_151" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_150" op: "Identity" input: "clip_by_global_norm/mul_151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_152" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_151" op: "Identity" input: "clip_by_global_norm/mul_152" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_153" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_152" op: "Identity" input: "clip_by_global_norm/mul_153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_154" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_153" op: "Identity" input: "clip_by_global_norm/mul_154" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_155" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_154" op: "Identity" input: "clip_by_global_norm/mul_155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_156" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_155" op: "Identity" input: "clip_by_global_norm/mul_156" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_157" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_156" op: "Identity" input: "clip_by_global_norm/mul_157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_158" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_157" op: "Identity" input: "clip_by_global_norm/mul_158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_159" op: "Mul" input: "gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_158" op: "Identity" input: "clip_by_global_norm/mul_159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/attention/output/layer_normalization_19/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_160" op: "Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_159" op: "Identity" input: "clip_by_global_norm/mul_160" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_161" op: "Mul" input: "gradients/bert/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_160" op: "Identity" input: "clip_by_global_norm/mul_161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_162" op: "Mul" input: "gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_161" op: "Identity" input: "clip_by_global_norm/mul_162" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_163" op: "Mul" input: "gradients/bert/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_162" op: "Identity" input: "clip_by_global_norm/mul_163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_164" op: "Mul" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_163" op: "Identity" input: "clip_by_global_norm/mul_164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_165" op: "Mul" input: "gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_164" op: "Identity" input: "clip_by_global_norm/mul_165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_9/output/layer_normalization_20/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_166" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_165" op: "Identity" input: "clip_by_global_norm/mul_166" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_167" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_166" op: "Identity" input: "clip_by_global_norm/mul_167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_168" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_167" op: "Identity" input: "clip_by_global_norm/mul_168" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_169" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_168" op: "Identity" input: "clip_by_global_norm/mul_169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_170" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_169" op: "Identity" input: "clip_by_global_norm/mul_170" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_171" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_170" op: "Identity" input: "clip_by_global_norm/mul_171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_172" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_171" op: "Identity" input: "clip_by_global_norm/mul_172" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_173" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_172" op: "Identity" input: "clip_by_global_norm/mul_173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_174" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_173" op: "Identity" input: "clip_by_global_norm/mul_174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_175" op: "Mul" input: "gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_174" op: "Identity" input: "clip_by_global_norm/mul_175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/attention/output/layer_normalization_21/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_176" op: "Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_175" op: "Identity" input: "clip_by_global_norm/mul_176" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_177" op: "Mul" input: "gradients/bert/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_176" op: "Identity" input: "clip_by_global_norm/mul_177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_178" op: "Mul" input: "gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_177" op: "Identity" input: "clip_by_global_norm/mul_178" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_179" op: "Mul" input: "gradients/bert/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_178" op: "Identity" input: "clip_by_global_norm/mul_179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_180" op: "Mul" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_179" op: "Identity" input: "clip_by_global_norm/mul_180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_181" op: "Mul" input: "gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_180" op: "Identity" input: "clip_by_global_norm/mul_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_10/output/layer_normalization_22/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_182" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_181" op: "Identity" input: "clip_by_global_norm/mul_182" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/query/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_183" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_182" op: "Identity" input: "clip_by_global_norm/mul_183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/query/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_184" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_183" op: "Identity" input: "clip_by_global_norm/mul_184" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/key/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_185" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_184" op: "Identity" input: "clip_by_global_norm/mul_185" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/key/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_186" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_185" op: "Identity" input: "clip_by_global_norm/mul_186" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/value/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_187" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_186" op: "Identity" input: "clip_by_global_norm/mul_187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/self/value/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_188" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_187" op: "Identity" input: "clip_by_global_norm/mul_188" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_189" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_188" op: "Identity" input: "clip_by_global_norm/mul_189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_190" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_189" op: "Identity" input: "clip_by_global_norm/mul_190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_191" op: "Mul" input: "gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_190" op: "Identity" input: "clip_by_global_norm/mul_191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/attention/output/layer_normalization_23/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_192" op: "Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_191" op: "Identity" input: "clip_by_global_norm/mul_192" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/intermediate/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_193" op: "Mul" input: "gradients/bert/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_192" op: "Identity" input: "clip_by_global_norm/mul_193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/intermediate/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "clip_by_global_norm/mul_194" op: "Mul" input: "gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_193" op: "Identity" input: "clip_by_global_norm/mul_194" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_195" op: "Mul" input: "gradients/bert/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_194" op: "Identity" input: "clip_by_global_norm/mul_195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_196" op: "Mul" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Reshape_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_195" op: "Identity" input: "clip_by_global_norm/mul_196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/mul_grad/Reshape_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_197" op: "Mul" input: "gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/Reshape" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_196" op: "Identity" input: "clip_by_global_norm/mul_197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/encoder/layer_11/output/layer_normalization_24/batchnorm/sub_grad/Reshape" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_198" op: "Mul" input: "gradients/bert/pooler/dense/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/pooler/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_197" op: "Identity" input: "clip_by_global_norm/mul_198" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/pooler/dense/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_199" op: "Mul" input: "gradients/bert/pooler/dense/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/pooler/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_198" op: "Identity" input: "clip_by_global_norm/mul_199" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/bert/pooler/dense/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_200" op: "Mul" input: "gradients/loss/MatMul_grad/MatMul_1" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/loss/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_199" op: "Identity" input: "clip_by_global_norm/mul_200" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/loss/MatMul_grad/MatMul_1" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "clip_by_global_norm/mul_201" op: "Mul" input: "gradients/loss/BiasAdd_grad/BiasAddGrad" input: "clip_by_global_norm/add" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/loss/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "clip_by_global_norm/clip_by_global_norm/_200" op: "Identity" input: "clip_by_global_norm/mul_201" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@gradients/loss/BiasAdd_grad/BiasAddGrad" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "bert/embeddings/word_embeddings/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\210R\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/word_embeddings/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/word_embeddings/adam_m/Initializer/zeros" op: "Fill" input: "bert/embeddings/word_embeddings/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/embeddings/word_embeddings/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/embeddings/word_embeddings/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 21128 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/embeddings/word_embeddings/adam_m" } } } node { name: "bert/embeddings/word_embeddings/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/embeddings/word_embeddings/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/word_embeddings/adam_m/Assign" op: "AssignVariableOp" input: "bert/embeddings/word_embeddings/adam_m" input: "bert/embeddings/word_embeddings/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/embeddings/word_embeddings/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/word_embeddings/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/word_embeddings/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\210R\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/word_embeddings/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/word_embeddings/adam_v/Initializer/zeros" op: "Fill" input: "bert/embeddings/word_embeddings/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/embeddings/word_embeddings/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/embeddings/word_embeddings/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/embeddings/word_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 21128 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/embeddings/word_embeddings/adam_v" } } } node { name: "bert/embeddings/word_embeddings/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/embeddings/word_embeddings/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/word_embeddings/adam_v/Assign" op: "AssignVariableOp" input: "bert/embeddings/word_embeddings/adam_v" input: "bert/embeddings/word_embeddings/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/embeddings/word_embeddings/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/word_embeddings/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_3/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_3/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/word_embeddings/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_3" op: "Mul" input: "Mul_3/x" input: "Mul_3/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "Mul_4/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_4/strided_slice/stack" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "Mul_4/strided_slice/stack_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "Mul_4/strided_slice/stack_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "Mul_4/strided_slice" op: "StridedSlice" input: "gradients/bert/embeddings/Gather_grad/Const" input: "Mul_4/strided_slice/stack" input: "Mul_4/strided_slice/stack_1" input: "Mul_4/strided_slice/stack_2" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 0 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 1 } } } node { name: "Mul_4/y" op: "UnsortedSegmentSum" input: "clip_by_global_norm/clip_by_global_norm/_0" input: "gradients/bert/embeddings/Gather_grad/Reshape_1" input: "Mul_4/strided_slice" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tindices" value { type: DT_INT32 } } attr { key: "Tnumsegments" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "Mul_4" op: "Mul" input: "Mul_4/x" input: "Mul_4/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "add_1" op: "AddV2" input: "Mul_3" input: "Mul_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "Mul_5/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_5/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/word_embeddings/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_5" op: "Mul" input: "Mul_5/x" input: "Mul_5/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "Square/strided_slice/stack" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "Square/strided_slice/stack_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "Square/strided_slice/stack_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "Square/strided_slice" op: "StridedSlice" input: "gradients/bert/embeddings/Gather_grad/Const" input: "Square/strided_slice/stack" input: "Square/strided_slice/stack_1" input: "Square/strided_slice/stack_2" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 0 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 1 } } } node { name: "Square/x" op: "UnsortedSegmentSum" input: "clip_by_global_norm/clip_by_global_norm/_0" input: "gradients/bert/embeddings/Gather_grad/Reshape_1" input: "Square/strided_slice" attr { key: "T" value { type: DT_FLOAT } } attr { key: "Tindices" value { type: DT_INT32 } } attr { key: "Tnumsegments" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "Square" op: "Square" input: "Square/x" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "Mul_6/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_6" op: "Mul" input: "Mul_6/x" input: "Square" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "add_2" op: "AddV2" input: "Mul_5" input: "Mul_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "Sqrt" op: "Sqrt" input: "add_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "add_3/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_3" op: "AddV2" input: "Sqrt" input: "add_3/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "truediv_1" op: "RealDiv" input: "add_1" input: "add_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/word_embeddings" attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_7/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_7" op: "Mul" input: "mul_7/x" input: "ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "add_4" op: "AddV2" input: "truediv_1" input: "mul_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "mul_8" op: "Mul" input: "add" input: "add_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_1" op: "ReadVariableOp" input: "bert/embeddings/word_embeddings" attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_1" op: "Sub" input: "ReadVariableOp_1" input: "mul_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_149" op: "AssignVariableOp" input: "bert/embeddings/word_embeddings" input: "sub_1" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_2" op: "ReadVariableOp" input: "bert/embeddings/word_embeddings" input: "^AssignVariableOp_149" attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_150" op: "AssignVariableOp" input: "bert/embeddings/word_embeddings/adam_m" input: "add_1" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_3" op: "ReadVariableOp" input: "bert/embeddings/word_embeddings/adam_m" input: "^AssignVariableOp_150" attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_151" op: "AssignVariableOp" input: "bert/embeddings/word_embeddings/adam_v" input: "add_2" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_4" op: "ReadVariableOp" input: "bert/embeddings/word_embeddings/adam_v" input: "^AssignVariableOp_151" attr { key: "_output_shapes" value { list { shape { dim { size: 21128 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/token_type_embeddings/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\002\000\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/token_type_embeddings/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/token_type_embeddings/adam_m/Initializer/zeros" op: "Fill" input: "bert/embeddings/token_type_embeddings/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/embeddings/token_type_embeddings/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/embeddings/token_type_embeddings/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 2 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/embeddings/token_type_embeddings/adam_m" } } } node { name: "bert/embeddings/token_type_embeddings/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/embeddings/token_type_embeddings/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/token_type_embeddings/adam_m/Assign" op: "AssignVariableOp" input: "bert/embeddings/token_type_embeddings/adam_m" input: "bert/embeddings/token_type_embeddings/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/embeddings/token_type_embeddings/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/token_type_embeddings/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/token_type_embeddings/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\002\000\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/token_type_embeddings/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/token_type_embeddings/adam_v/Initializer/zeros" op: "Fill" input: "bert/embeddings/token_type_embeddings/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/embeddings/token_type_embeddings/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/embeddings/token_type_embeddings/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/embeddings/token_type_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 2 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/embeddings/token_type_embeddings/adam_v" } } } node { name: "bert/embeddings/token_type_embeddings/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/embeddings/token_type_embeddings/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/token_type_embeddings/adam_v/Assign" op: "AssignVariableOp" input: "bert/embeddings/token_type_embeddings/adam_v" input: "bert/embeddings/token_type_embeddings/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/embeddings/token_type_embeddings/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/token_type_embeddings/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_9/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_9/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/token_type_embeddings/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_9" op: "Mul" input: "Mul_9/x" input: "Mul_9/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Mul_10/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_10" op: "Mul" input: "Mul_10/x" input: "clip_by_global_norm/clip_by_global_norm/_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "add_5" op: "AddV2" input: "Mul_9" input: "Mul_10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Mul_11/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_11/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/token_type_embeddings/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_11" op: "Mul" input: "Mul_11/x" input: "Mul_11/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Square_1" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Mul_12/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_12" op: "Mul" input: "Mul_12/x" input: "Square_1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "add_6" op: "AddV2" input: "Mul_11" input: "Mul_12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "Sqrt_1" op: "Sqrt" input: "add_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "add_7/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_7" op: "AddV2" input: "Sqrt_1" input: "add_7/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "truediv_2" op: "RealDiv" input: "add_5" input: "add_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_5" op: "ReadVariableOp" input: "bert/embeddings/token_type_embeddings" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_13/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_13" op: "Mul" input: "mul_13/x" input: "ReadVariableOp_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "add_8" op: "AddV2" input: "truediv_2" input: "mul_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "mul_14" op: "Mul" input: "add" input: "add_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_6" op: "ReadVariableOp" input: "bert/embeddings/token_type_embeddings" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_2" op: "Sub" input: "ReadVariableOp_6" input: "mul_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_152" op: "AssignVariableOp" input: "bert/embeddings/token_type_embeddings" input: "sub_2" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_7" op: "ReadVariableOp" input: "bert/embeddings/token_type_embeddings" input: "^AssignVariableOp_152" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_153" op: "AssignVariableOp" input: "bert/embeddings/token_type_embeddings/adam_m" input: "add_5" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_8" op: "ReadVariableOp" input: "bert/embeddings/token_type_embeddings/adam_m" input: "^AssignVariableOp_153" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_154" op: "AssignVariableOp" input: "bert/embeddings/token_type_embeddings/adam_v" input: "add_6" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_9" op: "ReadVariableOp" input: "bert/embeddings/token_type_embeddings/adam_v" input: "^AssignVariableOp_154" attr { key: "_output_shapes" value { list { shape { dim { size: 2 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/position_embeddings/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\002\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/position_embeddings/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/position_embeddings/adam_m/Initializer/zeros" op: "Fill" input: "bert/embeddings/position_embeddings/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/embeddings/position_embeddings/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/embeddings/position_embeddings/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 512 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/embeddings/position_embeddings/adam_m" } } } node { name: "bert/embeddings/position_embeddings/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/embeddings/position_embeddings/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/position_embeddings/adam_m/Assign" op: "AssignVariableOp" input: "bert/embeddings/position_embeddings/adam_m" input: "bert/embeddings/position_embeddings/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/embeddings/position_embeddings/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/position_embeddings/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/position_embeddings/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\002\000\000\000\003\000\000" } } } } node { name: "bert/embeddings/position_embeddings/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/embeddings/position_embeddings/adam_v/Initializer/zeros" op: "Fill" input: "bert/embeddings/position_embeddings/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/embeddings/position_embeddings/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/embeddings/position_embeddings/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/embeddings/position_embeddings/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 512 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/embeddings/position_embeddings/adam_v" } } } node { name: "bert/embeddings/position_embeddings/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/embeddings/position_embeddings/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/position_embeddings/adam_v/Assign" op: "AssignVariableOp" input: "bert/embeddings/position_embeddings/adam_v" input: "bert/embeddings/position_embeddings/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/embeddings/position_embeddings/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/position_embeddings/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_15/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_15/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/position_embeddings/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_15" op: "Mul" input: "Mul_15/x" input: "Mul_15/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "Mul_16/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_16" op: "Mul" input: "Mul_16/x" input: "clip_by_global_norm/clip_by_global_norm/_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "add_9" op: "AddV2" input: "Mul_15" input: "Mul_16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "Mul_17/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_17/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/position_embeddings/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_17" op: "Mul" input: "Mul_17/x" input: "Mul_17/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "Square_2" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "Mul_18/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_18" op: "Mul" input: "Mul_18/x" input: "Square_2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "add_10" op: "AddV2" input: "Mul_17" input: "Mul_18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "Sqrt_2" op: "Sqrt" input: "add_10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "add_11/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_11" op: "AddV2" input: "Sqrt_2" input: "add_11/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "truediv_3" op: "RealDiv" input: "add_9" input: "add_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_10" op: "ReadVariableOp" input: "bert/embeddings/position_embeddings" attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_19/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_19" op: "Mul" input: "mul_19/x" input: "ReadVariableOp_10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "add_12" op: "AddV2" input: "truediv_3" input: "mul_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "mul_20" op: "Mul" input: "add" input: "add_12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_11" op: "ReadVariableOp" input: "bert/embeddings/position_embeddings" attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_3" op: "Sub" input: "ReadVariableOp_11" input: "mul_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_155" op: "AssignVariableOp" input: "bert/embeddings/position_embeddings" input: "sub_3" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_12" op: "ReadVariableOp" input: "bert/embeddings/position_embeddings" input: "^AssignVariableOp_155" attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_156" op: "AssignVariableOp" input: "bert/embeddings/position_embeddings/adam_m" input: "add_9" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_13" op: "ReadVariableOp" input: "bert/embeddings/position_embeddings/adam_m" input: "^AssignVariableOp_156" attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_157" op: "AssignVariableOp" input: "bert/embeddings/position_embeddings/adam_v" input: "add_10" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_14" op: "ReadVariableOp" input: "bert/embeddings/position_embeddings/adam_v" input: "^AssignVariableOp_157" attr { key: "_output_shapes" value { list { shape { dim { size: 512 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/layer_normalization/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/layer_normalization/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/embeddings/layer_normalization/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/embeddings/layer_normalization/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/embeddings/layer_normalization/gamma/adam_m" } } } node { name: "bert/embeddings/layer_normalization/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/layer_normalization/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/gamma/adam_m" input: "bert/embeddings/layer_normalization/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/embeddings/layer_normalization/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/layer_normalization/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/layer_normalization/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/embeddings/layer_normalization/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/embeddings/layer_normalization/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/embeddings/layer_normalization/gamma/adam_v" } } } node { name: "bert/embeddings/layer_normalization/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/layer_normalization/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/gamma/adam_v" input: "bert/embeddings/layer_normalization/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/embeddings/layer_normalization/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_21/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_21/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_21" op: "Mul" input: "Mul_21/x" input: "Mul_21/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_22/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_22" op: "Mul" input: "Mul_22/x" input: "clip_by_global_norm/clip_by_global_norm/_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_13" op: "AddV2" input: "Mul_21" input: "Mul_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_23/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_23/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_23" op: "Mul" input: "Mul_23/x" input: "Mul_23/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_3" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_24/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_24" op: "Mul" input: "Mul_24/x" input: "Square_3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_14" op: "AddV2" input: "Mul_23" input: "Mul_24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_3" op: "Sqrt" input: "add_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_15/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_15" op: "AddV2" input: "Sqrt_3" input: "add_15/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_4" op: "RealDiv" input: "add_13" input: "add_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_25" op: "Mul" input: "add" input: "truediv_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_15" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_4" op: "Sub" input: "ReadVariableOp_15" input: "mul_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_158" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/gamma" input: "sub_4" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_16" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/gamma" input: "^AssignVariableOp_158" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_159" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/gamma/adam_m" input: "add_13" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_17" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/gamma/adam_m" input: "^AssignVariableOp_159" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_160" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/gamma/adam_v" input: "add_14" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_18" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/gamma/adam_v" input: "^AssignVariableOp_160" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/layer_normalization/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/layer_normalization/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/embeddings/layer_normalization/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/embeddings/layer_normalization/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/embeddings/layer_normalization/beta/adam_m" } } } node { name: "bert/embeddings/layer_normalization/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/layer_normalization/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/beta/adam_m" input: "bert/embeddings/layer_normalization/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/embeddings/layer_normalization/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/embeddings/layer_normalization/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/embeddings/layer_normalization/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/embeddings/layer_normalization/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/embeddings/layer_normalization/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/embeddings/layer_normalization/beta/adam_v" } } } node { name: "bert/embeddings/layer_normalization/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/embeddings/layer_normalization/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/beta/adam_v" input: "bert/embeddings/layer_normalization/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/embeddings/layer_normalization/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_26/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_26/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_26" op: "Mul" input: "Mul_26/x" input: "Mul_26/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_27/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_27" op: "Mul" input: "Mul_27/x" input: "clip_by_global_norm/clip_by_global_norm/_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_16" op: "AddV2" input: "Mul_26" input: "Mul_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_28/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_28/ReadVariableOp" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_28" op: "Mul" input: "Mul_28/x" input: "Mul_28/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_4" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_29/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_29" op: "Mul" input: "Mul_29/x" input: "Square_4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_17" op: "AddV2" input: "Mul_28" input: "Mul_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_4" op: "Sqrt" input: "add_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_18/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_18" op: "AddV2" input: "Sqrt_4" input: "add_18/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_5" op: "RealDiv" input: "add_16" input: "add_18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_30" op: "Mul" input: "add" input: "truediv_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_19" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_5" op: "Sub" input: "ReadVariableOp_19" input: "mul_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_161" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/beta" input: "sub_5" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_20" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/beta" input: "^AssignVariableOp_161" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_162" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/beta/adam_m" input: "add_16" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_21" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/beta/adam_m" input: "^AssignVariableOp_162" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_163" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/beta/adam_v" input: "add_17" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_22" op: "ReadVariableOp" input: "bert/embeddings/layer_normalization/beta/adam_v" input: "^AssignVariableOp_163" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_31/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_31/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_31" op: "Mul" input: "Mul_31/x" input: "Mul_31/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_32/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_32" op: "Mul" input: "Mul_32/x" input: "clip_by_global_norm/clip_by_global_norm/_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_19" op: "AddV2" input: "Mul_31" input: "Mul_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_33/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_33/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_33" op: "Mul" input: "Mul_33/x" input: "Mul_33/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_5" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_34/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_34" op: "Mul" input: "Mul_34/x" input: "Square_5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_20" op: "AddV2" input: "Mul_33" input: "Mul_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_5" op: "Sqrt" input: "add_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_21/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_21" op: "AddV2" input: "Sqrt_5" input: "add_21/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_6" op: "RealDiv" input: "add_19" input: "add_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_23" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_35/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_35" op: "Mul" input: "mul_35/x" input: "ReadVariableOp_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_22" op: "AddV2" input: "truediv_6" input: "mul_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_36" op: "Mul" input: "add" input: "add_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_24" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_6" op: "Sub" input: "ReadVariableOp_24" input: "mul_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_164" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel" input: "sub_6" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_25" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel" input: "^AssignVariableOp_164" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_165" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" input: "add_19" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_26" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" input: "^AssignVariableOp_165" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_166" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" input: "add_20" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_27" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" input: "^AssignVariableOp_166" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/query/bias/adam_m" } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/query/bias/adam_v" } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/query/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_37/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_37/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_37" op: "Mul" input: "Mul_37/x" input: "Mul_37/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_38/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_38" op: "Mul" input: "Mul_38/x" input: "clip_by_global_norm/clip_by_global_norm/_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_23" op: "AddV2" input: "Mul_37" input: "Mul_38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_39/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_39/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_39" op: "Mul" input: "Mul_39/x" input: "Mul_39/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_6" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_40/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_40" op: "Mul" input: "Mul_40/x" input: "Square_6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_24" op: "AddV2" input: "Mul_39" input: "Mul_40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_6" op: "Sqrt" input: "add_24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_25/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_25" op: "AddV2" input: "Sqrt_6" input: "add_25/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_7" op: "RealDiv" input: "add_23" input: "add_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_41" op: "Mul" input: "add" input: "truediv_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_28" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_7" op: "Sub" input: "ReadVariableOp_28" input: "mul_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_167" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias" input: "sub_7" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_29" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias" input: "^AssignVariableOp_167" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_168" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m" input: "add_23" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_30" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m" input: "^AssignVariableOp_168" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_169" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v" input: "add_24" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_31" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v" input: "^AssignVariableOp_169" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_42/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_42/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_42" op: "Mul" input: "Mul_42/x" input: "Mul_42/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_43/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_43" op: "Mul" input: "Mul_43/x" input: "clip_by_global_norm/clip_by_global_norm/_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_26" op: "AddV2" input: "Mul_42" input: "Mul_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_44/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_44/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_44" op: "Mul" input: "Mul_44/x" input: "Mul_44/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_7" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_45/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_45" op: "Mul" input: "Mul_45/x" input: "Square_7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_27" op: "AddV2" input: "Mul_44" input: "Mul_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_7" op: "Sqrt" input: "add_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_28/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_28" op: "AddV2" input: "Sqrt_7" input: "add_28/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_8" op: "RealDiv" input: "add_26" input: "add_28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_32" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_46/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_46" op: "Mul" input: "mul_46/x" input: "ReadVariableOp_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_29" op: "AddV2" input: "truediv_8" input: "mul_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_47" op: "Mul" input: "add" input: "add_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_33" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_8" op: "Sub" input: "ReadVariableOp_33" input: "mul_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_170" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel" input: "sub_8" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_34" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel" input: "^AssignVariableOp_170" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_171" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" input: "add_26" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_35" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" input: "^AssignVariableOp_171" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_172" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" input: "add_27" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_36" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" input: "^AssignVariableOp_172" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/key/bias/adam_m" } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/key/bias/adam_v" } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/key/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_48/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_48/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_48" op: "Mul" input: "Mul_48/x" input: "Mul_48/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_49/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_49" op: "Mul" input: "Mul_49/x" input: "clip_by_global_norm/clip_by_global_norm/_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_30" op: "AddV2" input: "Mul_48" input: "Mul_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_50/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_50/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_50" op: "Mul" input: "Mul_50/x" input: "Mul_50/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_8" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_51/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_51" op: "Mul" input: "Mul_51/x" input: "Square_8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_31" op: "AddV2" input: "Mul_50" input: "Mul_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_8" op: "Sqrt" input: "add_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_32/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_32" op: "AddV2" input: "Sqrt_8" input: "add_32/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_9" op: "RealDiv" input: "add_30" input: "add_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_52" op: "Mul" input: "add" input: "truediv_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_37" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_9" op: "Sub" input: "ReadVariableOp_37" input: "mul_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_173" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias" input: "sub_9" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_38" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias" input: "^AssignVariableOp_173" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_174" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m" input: "add_30" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_39" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m" input: "^AssignVariableOp_174" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_175" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v" input: "add_31" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_40" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v" input: "^AssignVariableOp_175" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_53/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_53/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_53" op: "Mul" input: "Mul_53/x" input: "Mul_53/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_54/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_54" op: "Mul" input: "Mul_54/x" input: "clip_by_global_norm/clip_by_global_norm/_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_33" op: "AddV2" input: "Mul_53" input: "Mul_54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_55/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_55/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_55" op: "Mul" input: "Mul_55/x" input: "Mul_55/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_9" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_56/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_56" op: "Mul" input: "Mul_56/x" input: "Square_9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_34" op: "AddV2" input: "Mul_55" input: "Mul_56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_9" op: "Sqrt" input: "add_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_35/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_35" op: "AddV2" input: "Sqrt_9" input: "add_35/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_10" op: "RealDiv" input: "add_33" input: "add_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_41" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_57/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_57" op: "Mul" input: "mul_57/x" input: "ReadVariableOp_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_36" op: "AddV2" input: "truediv_10" input: "mul_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_58" op: "Mul" input: "add" input: "add_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_42" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_10" op: "Sub" input: "ReadVariableOp_42" input: "mul_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_176" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel" input: "sub_10" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_43" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel" input: "^AssignVariableOp_176" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_177" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" input: "add_33" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_44" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" input: "^AssignVariableOp_177" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_178" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" input: "add_34" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_45" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" input: "^AssignVariableOp_178" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/value/bias/adam_m" } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/self/value/bias/adam_v" } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/self/value/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_59/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_59/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_59" op: "Mul" input: "Mul_59/x" input: "Mul_59/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_60/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_60" op: "Mul" input: "Mul_60/x" input: "clip_by_global_norm/clip_by_global_norm/_10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_37" op: "AddV2" input: "Mul_59" input: "Mul_60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_61/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_61/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_61" op: "Mul" input: "Mul_61/x" input: "Mul_61/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_10" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_62/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_62" op: "Mul" input: "Mul_62/x" input: "Square_10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_38" op: "AddV2" input: "Mul_61" input: "Mul_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_10" op: "Sqrt" input: "add_38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_39/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_39" op: "AddV2" input: "Sqrt_10" input: "add_39/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_11" op: "RealDiv" input: "add_37" input: "add_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_63" op: "Mul" input: "add" input: "truediv_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_46" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_11" op: "Sub" input: "ReadVariableOp_46" input: "mul_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_179" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias" input: "sub_11" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_47" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias" input: "^AssignVariableOp_179" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_180" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m" input: "add_37" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_48" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m" input: "^AssignVariableOp_180" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_181" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v" input: "add_38" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_49" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v" input: "^AssignVariableOp_181" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_64/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_64/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_64" op: "Mul" input: "Mul_64/x" input: "Mul_64/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_65/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_65" op: "Mul" input: "Mul_65/x" input: "clip_by_global_norm/clip_by_global_norm/_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_40" op: "AddV2" input: "Mul_64" input: "Mul_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_66/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_66/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_66" op: "Mul" input: "Mul_66/x" input: "Mul_66/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_11" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_67/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_67" op: "Mul" input: "Mul_67/x" input: "Square_11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_41" op: "AddV2" input: "Mul_66" input: "Mul_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_11" op: "Sqrt" input: "add_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_42/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_42" op: "AddV2" input: "Sqrt_11" input: "add_42/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_12" op: "RealDiv" input: "add_40" input: "add_42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_50" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_68/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_68" op: "Mul" input: "mul_68/x" input: "ReadVariableOp_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_43" op: "AddV2" input: "truediv_12" input: "mul_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_69" op: "Mul" input: "add" input: "add_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_51" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_12" op: "Sub" input: "ReadVariableOp_51" input: "mul_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_182" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel" input: "sub_12" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_52" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel" input: "^AssignVariableOp_182" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_183" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" input: "add_40" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_53" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" input: "^AssignVariableOp_183" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_184" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" input: "add_41" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_54" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" input: "^AssignVariableOp_184" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_70/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_70/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_70" op: "Mul" input: "Mul_70/x" input: "Mul_70/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_71/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_71" op: "Mul" input: "Mul_71/x" input: "clip_by_global_norm/clip_by_global_norm/_12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_44" op: "AddV2" input: "Mul_70" input: "Mul_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_72/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_72/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_72" op: "Mul" input: "Mul_72/x" input: "Mul_72/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_12" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_73/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_73" op: "Mul" input: "Mul_73/x" input: "Square_12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_45" op: "AddV2" input: "Mul_72" input: "Mul_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_12" op: "Sqrt" input: "add_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_46/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_46" op: "AddV2" input: "Sqrt_12" input: "add_46/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_13" op: "RealDiv" input: "add_44" input: "add_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_74" op: "Mul" input: "add" input: "truediv_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_55" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_13" op: "Sub" input: "ReadVariableOp_55" input: "mul_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_185" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias" input: "sub_13" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_56" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias" input: "^AssignVariableOp_185" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_186" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" input: "add_44" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_57" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" input: "^AssignVariableOp_186" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_187" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" input: "add_45" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_58" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" input: "^AssignVariableOp_187" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_75/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_75/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_75" op: "Mul" input: "Mul_75/x" input: "Mul_75/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_76/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_76" op: "Mul" input: "Mul_76/x" input: "clip_by_global_norm/clip_by_global_norm/_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_47" op: "AddV2" input: "Mul_75" input: "Mul_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_77/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_77/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_77" op: "Mul" input: "Mul_77/x" input: "Mul_77/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_13" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_78/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_78" op: "Mul" input: "Mul_78/x" input: "Square_13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_48" op: "AddV2" input: "Mul_77" input: "Mul_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_13" op: "Sqrt" input: "add_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_49/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_49" op: "AddV2" input: "Sqrt_13" input: "add_49/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_14" op: "RealDiv" input: "add_47" input: "add_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_79" op: "Mul" input: "add" input: "truediv_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_59" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_14" op: "Sub" input: "ReadVariableOp_59" input: "mul_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_188" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" input: "sub_14" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_60" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" input: "^AssignVariableOp_188" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_189" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" input: "add_47" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_61" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" input: "^AssignVariableOp_189" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_190" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" input: "add_48" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_62" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" input: "^AssignVariableOp_190" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_80/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_80/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_80" op: "Mul" input: "Mul_80/x" input: "Mul_80/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_81/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_81" op: "Mul" input: "Mul_81/x" input: "clip_by_global_norm/clip_by_global_norm/_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_50" op: "AddV2" input: "Mul_80" input: "Mul_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_82/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_82/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_82" op: "Mul" input: "Mul_82/x" input: "Mul_82/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_14" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_83/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_83" op: "Mul" input: "Mul_83/x" input: "Square_14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_51" op: "AddV2" input: "Mul_82" input: "Mul_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_14" op: "Sqrt" input: "add_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_52/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_52" op: "AddV2" input: "Sqrt_14" input: "add_52/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_15" op: "RealDiv" input: "add_50" input: "add_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_84" op: "Mul" input: "add" input: "truediv_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_63" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_15" op: "Sub" input: "ReadVariableOp_63" input: "mul_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_191" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta" input: "sub_15" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_64" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta" input: "^AssignVariableOp_191" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_192" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" input: "add_50" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_65" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" input: "^AssignVariableOp_192" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_193" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" input: "add_51" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_66" op: "ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" input: "^AssignVariableOp_193" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_85/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_85/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_85" op: "Mul" input: "Mul_85/x" input: "Mul_85/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_86/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_86" op: "Mul" input: "Mul_86/x" input: "clip_by_global_norm/clip_by_global_norm/_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_53" op: "AddV2" input: "Mul_85" input: "Mul_86" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_87/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_87/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_87" op: "Mul" input: "Mul_87/x" input: "Mul_87/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_15" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_88/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_88" op: "Mul" input: "Mul_88/x" input: "Square_15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_54" op: "AddV2" input: "Mul_87" input: "Mul_88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_15" op: "Sqrt" input: "add_54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_55/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_55" op: "AddV2" input: "Sqrt_15" input: "add_55/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_16" op: "RealDiv" input: "add_53" input: "add_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_67" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_89/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_89" op: "Mul" input: "mul_89/x" input: "ReadVariableOp_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_56" op: "AddV2" input: "truediv_16" input: "mul_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_90" op: "Mul" input: "add" input: "add_56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_68" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_16" op: "Sub" input: "ReadVariableOp_68" input: "mul_90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "AssignVariableOp_194" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel" input: "sub_16" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_69" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel" input: "^AssignVariableOp_194" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_195" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" input: "add_53" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_70" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" input: "^AssignVariableOp_195" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_196" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" input: "add_54" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_71" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" input: "^AssignVariableOp_196" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_91/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_91/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_91" op: "Mul" input: "Mul_91/x" input: "Mul_91/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_92/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_92" op: "Mul" input: "Mul_92/x" input: "clip_by_global_norm/clip_by_global_norm/_16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_57" op: "AddV2" input: "Mul_91" input: "Mul_92" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_93/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_93/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_93" op: "Mul" input: "Mul_93/x" input: "Mul_93/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_16" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_94/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_94" op: "Mul" input: "Mul_94/x" input: "Square_16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_58" op: "AddV2" input: "Mul_93" input: "Mul_94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_16" op: "Sqrt" input: "add_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_59/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_59" op: "AddV2" input: "Sqrt_16" input: "add_59/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_17" op: "RealDiv" input: "add_57" input: "add_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_95" op: "Mul" input: "add" input: "truediv_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "ReadVariableOp_72" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_17" op: "Sub" input: "ReadVariableOp_72" input: "mul_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "AssignVariableOp_197" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias" input: "sub_17" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_73" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias" input: "^AssignVariableOp_197" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_198" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" input: "add_57" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_74" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" input: "^AssignVariableOp_198" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_199" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" input: "add_58" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_75" op: "ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" input: "^AssignVariableOp_199" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_m" input: "bert/encoder/layer_0/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_v" input: "bert/encoder/layer_0/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_96/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_96/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_96" op: "Mul" input: "Mul_96/x" input: "Mul_96/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_97/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_97" op: "Mul" input: "Mul_97/x" input: "clip_by_global_norm/clip_by_global_norm/_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_60" op: "AddV2" input: "Mul_96" input: "Mul_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_98/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_98/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_98" op: "Mul" input: "Mul_98/x" input: "Mul_98/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_17" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_99/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_99" op: "Mul" input: "Mul_99/x" input: "Square_17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_61" op: "AddV2" input: "Mul_98" input: "Mul_99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_17" op: "Sqrt" input: "add_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_62/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_62" op: "AddV2" input: "Sqrt_17" input: "add_62/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_18" op: "RealDiv" input: "add_60" input: "add_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_76" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_100/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_100" op: "Mul" input: "mul_100/x" input: "ReadVariableOp_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_63" op: "AddV2" input: "truediv_18" input: "mul_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_101" op: "Mul" input: "add" input: "add_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_77" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_18" op: "Sub" input: "ReadVariableOp_77" input: "mul_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_200" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/kernel" input: "sub_18" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_78" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/kernel" input: "^AssignVariableOp_200" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_201" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_m" input: "add_60" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_79" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_m" input: "^AssignVariableOp_201" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_202" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_v" input: "add_61" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_80" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_v" input: "^AssignVariableOp_202" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/bias/adam_m" input: "bert/encoder/layer_0/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/bias/adam_v" input: "bert/encoder/layer_0/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_102/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_102/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_102" op: "Mul" input: "Mul_102/x" input: "Mul_102/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_103/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_103" op: "Mul" input: "Mul_103/x" input: "clip_by_global_norm/clip_by_global_norm/_18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_64" op: "AddV2" input: "Mul_102" input: "Mul_103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_104/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_104/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_104" op: "Mul" input: "Mul_104/x" input: "Mul_104/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_18" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_105/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_105" op: "Mul" input: "Mul_105/x" input: "Square_18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_65" op: "AddV2" input: "Mul_104" input: "Mul_105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_18" op: "Sqrt" input: "add_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_66/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_66" op: "AddV2" input: "Sqrt_18" input: "add_66/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_19" op: "RealDiv" input: "add_64" input: "add_66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_106" op: "Mul" input: "add" input: "truediv_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_81" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_19" op: "Sub" input: "ReadVariableOp_81" input: "mul_106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_203" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/bias" input: "sub_19" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_82" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/bias" input: "^AssignVariableOp_203" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_204" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/bias/adam_m" input: "add_64" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_83" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/bias/adam_m" input: "^AssignVariableOp_204" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_205" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/bias/adam_v" input: "add_65" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_84" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/dense/bias/adam_v" input: "^AssignVariableOp_205" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_107/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_107/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_107" op: "Mul" input: "Mul_107/x" input: "Mul_107/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_108/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_108" op: "Mul" input: "Mul_108/x" input: "clip_by_global_norm/clip_by_global_norm/_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_67" op: "AddV2" input: "Mul_107" input: "Mul_108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_109/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_109/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_109" op: "Mul" input: "Mul_109/x" input: "Mul_109/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_19" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_110/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_110" op: "Mul" input: "Mul_110/x" input: "Square_19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_68" op: "AddV2" input: "Mul_109" input: "Mul_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_19" op: "Sqrt" input: "add_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_69/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_69" op: "AddV2" input: "Sqrt_19" input: "add_69/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_20" op: "RealDiv" input: "add_67" input: "add_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_111" op: "Mul" input: "add" input: "truediv_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_85" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_20" op: "Sub" input: "ReadVariableOp_85" input: "mul_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_206" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma" input: "sub_20" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_86" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma" input: "^AssignVariableOp_206" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_207" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" input: "add_67" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_87" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" input: "^AssignVariableOp_207" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_208" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" input: "add_68" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_88" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" input: "^AssignVariableOp_208" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_112/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_112/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_112" op: "Mul" input: "Mul_112/x" input: "Mul_112/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_113/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_113" op: "Mul" input: "Mul_113/x" input: "clip_by_global_norm/clip_by_global_norm/_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_70" op: "AddV2" input: "Mul_112" input: "Mul_113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_114/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_114/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_114" op: "Mul" input: "Mul_114/x" input: "Mul_114/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_20" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_115/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_115" op: "Mul" input: "Mul_115/x" input: "Square_20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_71" op: "AddV2" input: "Mul_114" input: "Mul_115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_20" op: "Sqrt" input: "add_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_72/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_72" op: "AddV2" input: "Sqrt_20" input: "add_72/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_21" op: "RealDiv" input: "add_70" input: "add_72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_116" op: "Mul" input: "add" input: "truediv_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_89" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_21" op: "Sub" input: "ReadVariableOp_89" input: "mul_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_209" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta" input: "sub_21" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_90" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta" input: "^AssignVariableOp_209" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_210" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" input: "add_70" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_91" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" input: "^AssignVariableOp_210" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_211" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" input: "add_71" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_92" op: "ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" input: "^AssignVariableOp_211" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_117/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_117/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_117" op: "Mul" input: "Mul_117/x" input: "Mul_117/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_118/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_118" op: "Mul" input: "Mul_118/x" input: "clip_by_global_norm/clip_by_global_norm/_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_73" op: "AddV2" input: "Mul_117" input: "Mul_118" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_119/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_119/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_119" op: "Mul" input: "Mul_119/x" input: "Mul_119/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_21" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_120/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_120" op: "Mul" input: "Mul_120/x" input: "Square_21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_74" op: "AddV2" input: "Mul_119" input: "Mul_120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_21" op: "Sqrt" input: "add_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_75/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_75" op: "AddV2" input: "Sqrt_21" input: "add_75/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_22" op: "RealDiv" input: "add_73" input: "add_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_93" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_121/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_121" op: "Mul" input: "mul_121/x" input: "ReadVariableOp_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_76" op: "AddV2" input: "truediv_22" input: "mul_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_122" op: "Mul" input: "add" input: "add_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_94" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_22" op: "Sub" input: "ReadVariableOp_94" input: "mul_122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_212" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel" input: "sub_22" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_95" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel" input: "^AssignVariableOp_212" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_213" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" input: "add_73" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_96" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" input: "^AssignVariableOp_213" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_214" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" input: "add_74" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_97" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" input: "^AssignVariableOp_214" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/query/bias/adam_m" } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/query/bias/adam_v" } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/query/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_123/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_123/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_123" op: "Mul" input: "Mul_123/x" input: "Mul_123/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_124/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_124" op: "Mul" input: "Mul_124/x" input: "clip_by_global_norm/clip_by_global_norm/_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_77" op: "AddV2" input: "Mul_123" input: "Mul_124" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_125/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_125/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_125" op: "Mul" input: "Mul_125/x" input: "Mul_125/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_22" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_126/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_126" op: "Mul" input: "Mul_126/x" input: "Square_22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_78" op: "AddV2" input: "Mul_125" input: "Mul_126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_22" op: "Sqrt" input: "add_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_79/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_79" op: "AddV2" input: "Sqrt_22" input: "add_79/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_23" op: "RealDiv" input: "add_77" input: "add_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_127" op: "Mul" input: "add" input: "truediv_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_98" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_23" op: "Sub" input: "ReadVariableOp_98" input: "mul_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_215" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias" input: "sub_23" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_99" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias" input: "^AssignVariableOp_215" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_216" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m" input: "add_77" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_100" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m" input: "^AssignVariableOp_216" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_217" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v" input: "add_78" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_101" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v" input: "^AssignVariableOp_217" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_128/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_128/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_128" op: "Mul" input: "Mul_128/x" input: "Mul_128/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_129/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_129" op: "Mul" input: "Mul_129/x" input: "clip_by_global_norm/clip_by_global_norm/_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_80" op: "AddV2" input: "Mul_128" input: "Mul_129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_130/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_130/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_130" op: "Mul" input: "Mul_130/x" input: "Mul_130/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_23" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_131/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_131" op: "Mul" input: "Mul_131/x" input: "Square_23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_81" op: "AddV2" input: "Mul_130" input: "Mul_131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_23" op: "Sqrt" input: "add_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_82/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_82" op: "AddV2" input: "Sqrt_23" input: "add_82/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_24" op: "RealDiv" input: "add_80" input: "add_82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_102" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_132/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_132" op: "Mul" input: "mul_132/x" input: "ReadVariableOp_102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_83" op: "AddV2" input: "truediv_24" input: "mul_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_133" op: "Mul" input: "add" input: "add_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_103" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_24" op: "Sub" input: "ReadVariableOp_103" input: "mul_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_218" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel" input: "sub_24" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_104" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel" input: "^AssignVariableOp_218" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_219" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" input: "add_80" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_105" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" input: "^AssignVariableOp_219" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_220" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" input: "add_81" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_106" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" input: "^AssignVariableOp_220" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/key/bias/adam_m" } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/key/bias/adam_v" } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/key/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_134/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_134/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_134" op: "Mul" input: "Mul_134/x" input: "Mul_134/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_135/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_135" op: "Mul" input: "Mul_135/x" input: "clip_by_global_norm/clip_by_global_norm/_24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_84" op: "AddV2" input: "Mul_134" input: "Mul_135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_136/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_136/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_136" op: "Mul" input: "Mul_136/x" input: "Mul_136/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_24" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_137/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_137" op: "Mul" input: "Mul_137/x" input: "Square_24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_85" op: "AddV2" input: "Mul_136" input: "Mul_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_24" op: "Sqrt" input: "add_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_86/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_86" op: "AddV2" input: "Sqrt_24" input: "add_86/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_25" op: "RealDiv" input: "add_84" input: "add_86" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_138" op: "Mul" input: "add" input: "truediv_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_107" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_25" op: "Sub" input: "ReadVariableOp_107" input: "mul_138" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_221" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias" input: "sub_25" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_108" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias" input: "^AssignVariableOp_221" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_222" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m" input: "add_84" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_109" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m" input: "^AssignVariableOp_222" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_223" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v" input: "add_85" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_110" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v" input: "^AssignVariableOp_223" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_139/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_139/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_139" op: "Mul" input: "Mul_139/x" input: "Mul_139/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_140/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_140" op: "Mul" input: "Mul_140/x" input: "clip_by_global_norm/clip_by_global_norm/_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_87" op: "AddV2" input: "Mul_139" input: "Mul_140" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_141/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_141/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_141" op: "Mul" input: "Mul_141/x" input: "Mul_141/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_25" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_142/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_142" op: "Mul" input: "Mul_142/x" input: "Square_25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_88" op: "AddV2" input: "Mul_141" input: "Mul_142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_25" op: "Sqrt" input: "add_88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_89/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_89" op: "AddV2" input: "Sqrt_25" input: "add_89/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_26" op: "RealDiv" input: "add_87" input: "add_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_111" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_143/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_143" op: "Mul" input: "mul_143/x" input: "ReadVariableOp_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_90" op: "AddV2" input: "truediv_26" input: "mul_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_144" op: "Mul" input: "add" input: "add_90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_112" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_26" op: "Sub" input: "ReadVariableOp_112" input: "mul_144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_224" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel" input: "sub_26" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_113" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel" input: "^AssignVariableOp_224" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_225" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" input: "add_87" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_114" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" input: "^AssignVariableOp_225" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_226" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" input: "add_88" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_115" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" input: "^AssignVariableOp_226" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/value/bias/adam_m" } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/self/value/bias/adam_v" } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/self/value/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_145/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_145/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_145" op: "Mul" input: "Mul_145/x" input: "Mul_145/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_146/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_146" op: "Mul" input: "Mul_146/x" input: "clip_by_global_norm/clip_by_global_norm/_26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_91" op: "AddV2" input: "Mul_145" input: "Mul_146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_147/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_147/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_147" op: "Mul" input: "Mul_147/x" input: "Mul_147/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_26" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_148/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_148" op: "Mul" input: "Mul_148/x" input: "Square_26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_92" op: "AddV2" input: "Mul_147" input: "Mul_148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_26" op: "Sqrt" input: "add_92" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_93/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_93" op: "AddV2" input: "Sqrt_26" input: "add_93/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_27" op: "RealDiv" input: "add_91" input: "add_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_149" op: "Mul" input: "add" input: "truediv_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_116" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_27" op: "Sub" input: "ReadVariableOp_116" input: "mul_149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_227" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias" input: "sub_27" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_117" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias" input: "^AssignVariableOp_227" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_228" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m" input: "add_91" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_118" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m" input: "^AssignVariableOp_228" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_229" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v" input: "add_92" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_119" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v" input: "^AssignVariableOp_229" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_150/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_150/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_150" op: "Mul" input: "Mul_150/x" input: "Mul_150/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_151/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_151" op: "Mul" input: "Mul_151/x" input: "clip_by_global_norm/clip_by_global_norm/_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_94" op: "AddV2" input: "Mul_150" input: "Mul_151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_152/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_152/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_152" op: "Mul" input: "Mul_152/x" input: "Mul_152/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_27" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_153/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_153" op: "Mul" input: "Mul_153/x" input: "Square_27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_95" op: "AddV2" input: "Mul_152" input: "Mul_153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_27" op: "Sqrt" input: "add_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_96/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_96" op: "AddV2" input: "Sqrt_27" input: "add_96/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_28" op: "RealDiv" input: "add_94" input: "add_96" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_120" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_154/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_154" op: "Mul" input: "mul_154/x" input: "ReadVariableOp_120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_97" op: "AddV2" input: "truediv_28" input: "mul_154" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_155" op: "Mul" input: "add" input: "add_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_121" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_28" op: "Sub" input: "ReadVariableOp_121" input: "mul_155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_230" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel" input: "sub_28" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_122" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel" input: "^AssignVariableOp_230" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_231" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" input: "add_94" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_123" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" input: "^AssignVariableOp_231" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_232" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" input: "add_95" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_124" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" input: "^AssignVariableOp_232" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_156/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_156/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_156" op: "Mul" input: "Mul_156/x" input: "Mul_156/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_157/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_157" op: "Mul" input: "Mul_157/x" input: "clip_by_global_norm/clip_by_global_norm/_28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_98" op: "AddV2" input: "Mul_156" input: "Mul_157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_158/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_158/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_158" op: "Mul" input: "Mul_158/x" input: "Mul_158/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_28" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_159/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_159" op: "Mul" input: "Mul_159/x" input: "Square_28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_99" op: "AddV2" input: "Mul_158" input: "Mul_159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_28" op: "Sqrt" input: "add_99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_100/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_100" op: "AddV2" input: "Sqrt_28" input: "add_100/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_29" op: "RealDiv" input: "add_98" input: "add_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_160" op: "Mul" input: "add" input: "truediv_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_125" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_29" op: "Sub" input: "ReadVariableOp_125" input: "mul_160" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_233" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias" input: "sub_29" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_126" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias" input: "^AssignVariableOp_233" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_234" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" input: "add_98" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_127" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" input: "^AssignVariableOp_234" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_235" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" input: "add_99" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_128" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" input: "^AssignVariableOp_235" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_161/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_161/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_161" op: "Mul" input: "Mul_161/x" input: "Mul_161/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_162/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_162" op: "Mul" input: "Mul_162/x" input: "clip_by_global_norm/clip_by_global_norm/_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_101" op: "AddV2" input: "Mul_161" input: "Mul_162" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_163/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_163/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_163" op: "Mul" input: "Mul_163/x" input: "Mul_163/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_29" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_164/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_164" op: "Mul" input: "Mul_164/x" input: "Square_29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_102" op: "AddV2" input: "Mul_163" input: "Mul_164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_29" op: "Sqrt" input: "add_102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_103/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_103" op: "AddV2" input: "Sqrt_29" input: "add_103/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_30" op: "RealDiv" input: "add_101" input: "add_103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_165" op: "Mul" input: "add" input: "truediv_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_129" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_30" op: "Sub" input: "ReadVariableOp_129" input: "mul_165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_236" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" input: "sub_30" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_130" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" input: "^AssignVariableOp_236" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_237" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" input: "add_101" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_131" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" input: "^AssignVariableOp_237" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_238" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" input: "add_102" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_132" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" input: "^AssignVariableOp_238" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_166/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_166/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_166" op: "Mul" input: "Mul_166/x" input: "Mul_166/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_167/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_167" op: "Mul" input: "Mul_167/x" input: "clip_by_global_norm/clip_by_global_norm/_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_104" op: "AddV2" input: "Mul_166" input: "Mul_167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_168/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_168/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_168" op: "Mul" input: "Mul_168/x" input: "Mul_168/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_30" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_169/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_169" op: "Mul" input: "Mul_169/x" input: "Square_30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_105" op: "AddV2" input: "Mul_168" input: "Mul_169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_30" op: "Sqrt" input: "add_105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_106/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_106" op: "AddV2" input: "Sqrt_30" input: "add_106/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_31" op: "RealDiv" input: "add_104" input: "add_106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_170" op: "Mul" input: "add" input: "truediv_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_133" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_31" op: "Sub" input: "ReadVariableOp_133" input: "mul_170" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_239" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta" input: "sub_31" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_134" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta" input: "^AssignVariableOp_239" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_240" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" input: "add_104" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_135" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" input: "^AssignVariableOp_240" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_241" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" input: "add_105" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_136" op: "ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" input: "^AssignVariableOp_241" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_171/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_171/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_171" op: "Mul" input: "Mul_171/x" input: "Mul_171/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_172/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_172" op: "Mul" input: "Mul_172/x" input: "clip_by_global_norm/clip_by_global_norm/_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_107" op: "AddV2" input: "Mul_171" input: "Mul_172" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_173/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_173/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_173" op: "Mul" input: "Mul_173/x" input: "Mul_173/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_31" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_174/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_174" op: "Mul" input: "Mul_174/x" input: "Square_31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_108" op: "AddV2" input: "Mul_173" input: "Mul_174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_31" op: "Sqrt" input: "add_108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_109/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_109" op: "AddV2" input: "Sqrt_31" input: "add_109/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_32" op: "RealDiv" input: "add_107" input: "add_109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_137" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_175/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_175" op: "Mul" input: "mul_175/x" input: "ReadVariableOp_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_110" op: "AddV2" input: "truediv_32" input: "mul_175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_176" op: "Mul" input: "add" input: "add_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_138" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_32" op: "Sub" input: "ReadVariableOp_138" input: "mul_176" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "AssignVariableOp_242" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel" input: "sub_32" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_139" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel" input: "^AssignVariableOp_242" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_243" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" input: "add_107" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_140" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" input: "^AssignVariableOp_243" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_244" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" input: "add_108" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_141" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" input: "^AssignVariableOp_244" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_177/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_177/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_177" op: "Mul" input: "Mul_177/x" input: "Mul_177/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_178/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_178" op: "Mul" input: "Mul_178/x" input: "clip_by_global_norm/clip_by_global_norm/_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_111" op: "AddV2" input: "Mul_177" input: "Mul_178" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_179/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_179/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_179" op: "Mul" input: "Mul_179/x" input: "Mul_179/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_32" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_180/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_180" op: "Mul" input: "Mul_180/x" input: "Square_32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_112" op: "AddV2" input: "Mul_179" input: "Mul_180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_32" op: "Sqrt" input: "add_112" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_113/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_113" op: "AddV2" input: "Sqrt_32" input: "add_113/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_33" op: "RealDiv" input: "add_111" input: "add_113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_181" op: "Mul" input: "add" input: "truediv_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "ReadVariableOp_142" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_33" op: "Sub" input: "ReadVariableOp_142" input: "mul_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "AssignVariableOp_245" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias" input: "sub_33" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_143" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias" input: "^AssignVariableOp_245" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_246" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" input: "add_111" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_144" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" input: "^AssignVariableOp_246" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_247" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" input: "add_112" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_145" op: "ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" input: "^AssignVariableOp_247" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_m" input: "bert/encoder/layer_1/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_v" input: "bert/encoder/layer_1/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_182/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_182/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_182" op: "Mul" input: "Mul_182/x" input: "Mul_182/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_183/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_183" op: "Mul" input: "Mul_183/x" input: "clip_by_global_norm/clip_by_global_norm/_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_114" op: "AddV2" input: "Mul_182" input: "Mul_183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_184/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_184/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_184" op: "Mul" input: "Mul_184/x" input: "Mul_184/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_33" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_185/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_185" op: "Mul" input: "Mul_185/x" input: "Square_33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_115" op: "AddV2" input: "Mul_184" input: "Mul_185" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_33" op: "Sqrt" input: "add_115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_116/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_116" op: "AddV2" input: "Sqrt_33" input: "add_116/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_34" op: "RealDiv" input: "add_114" input: "add_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_146" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_186/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_186" op: "Mul" input: "mul_186/x" input: "ReadVariableOp_146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_117" op: "AddV2" input: "truediv_34" input: "mul_186" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_187" op: "Mul" input: "add" input: "add_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_147" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_34" op: "Sub" input: "ReadVariableOp_147" input: "mul_187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_248" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/kernel" input: "sub_34" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_148" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/kernel" input: "^AssignVariableOp_248" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_249" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_m" input: "add_114" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_149" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_m" input: "^AssignVariableOp_249" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_250" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_v" input: "add_115" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_150" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_v" input: "^AssignVariableOp_250" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/bias/adam_m" input: "bert/encoder/layer_1/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/bias/adam_v" input: "bert/encoder/layer_1/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_188/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_188/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_188" op: "Mul" input: "Mul_188/x" input: "Mul_188/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_189/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_189" op: "Mul" input: "Mul_189/x" input: "clip_by_global_norm/clip_by_global_norm/_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_118" op: "AddV2" input: "Mul_188" input: "Mul_189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_190/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_190/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_190" op: "Mul" input: "Mul_190/x" input: "Mul_190/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_34" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_191/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_191" op: "Mul" input: "Mul_191/x" input: "Square_34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_119" op: "AddV2" input: "Mul_190" input: "Mul_191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_34" op: "Sqrt" input: "add_119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_120/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_120" op: "AddV2" input: "Sqrt_34" input: "add_120/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_35" op: "RealDiv" input: "add_118" input: "add_120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_192" op: "Mul" input: "add" input: "truediv_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_151" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_35" op: "Sub" input: "ReadVariableOp_151" input: "mul_192" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_251" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/bias" input: "sub_35" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_152" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/bias" input: "^AssignVariableOp_251" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_252" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/bias/adam_m" input: "add_118" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_153" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/bias/adam_m" input: "^AssignVariableOp_252" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_253" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/bias/adam_v" input: "add_119" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_154" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/dense/bias/adam_v" input: "^AssignVariableOp_253" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_193/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_193/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_193" op: "Mul" input: "Mul_193/x" input: "Mul_193/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_194/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_194" op: "Mul" input: "Mul_194/x" input: "clip_by_global_norm/clip_by_global_norm/_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_121" op: "AddV2" input: "Mul_193" input: "Mul_194" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_195/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_195/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_195" op: "Mul" input: "Mul_195/x" input: "Mul_195/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_35" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_196/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_196" op: "Mul" input: "Mul_196/x" input: "Square_35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_122" op: "AddV2" input: "Mul_195" input: "Mul_196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_35" op: "Sqrt" input: "add_122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_123/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_123" op: "AddV2" input: "Sqrt_35" input: "add_123/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_36" op: "RealDiv" input: "add_121" input: "add_123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_197" op: "Mul" input: "add" input: "truediv_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_155" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_36" op: "Sub" input: "ReadVariableOp_155" input: "mul_197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_254" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma" input: "sub_36" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_156" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma" input: "^AssignVariableOp_254" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_255" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" input: "add_121" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_157" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" input: "^AssignVariableOp_255" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_256" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" input: "add_122" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_158" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" input: "^AssignVariableOp_256" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_198/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_198/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_198" op: "Mul" input: "Mul_198/x" input: "Mul_198/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_199/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_199" op: "Mul" input: "Mul_199/x" input: "clip_by_global_norm/clip_by_global_norm/_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_124" op: "AddV2" input: "Mul_198" input: "Mul_199" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_200/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_200/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_200" op: "Mul" input: "Mul_200/x" input: "Mul_200/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_36" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_201/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_201" op: "Mul" input: "Mul_201/x" input: "Square_36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_125" op: "AddV2" input: "Mul_200" input: "Mul_201" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_36" op: "Sqrt" input: "add_125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_126/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_126" op: "AddV2" input: "Sqrt_36" input: "add_126/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_37" op: "RealDiv" input: "add_124" input: "add_126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_202" op: "Mul" input: "add" input: "truediv_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_159" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_37" op: "Sub" input: "ReadVariableOp_159" input: "mul_202" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_257" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta" input: "sub_37" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_160" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta" input: "^AssignVariableOp_257" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_258" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" input: "add_124" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_161" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" input: "^AssignVariableOp_258" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_259" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" input: "add_125" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_162" op: "ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" input: "^AssignVariableOp_259" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_203/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_203/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_203" op: "Mul" input: "Mul_203/x" input: "Mul_203/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_204/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_204" op: "Mul" input: "Mul_204/x" input: "clip_by_global_norm/clip_by_global_norm/_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_127" op: "AddV2" input: "Mul_203" input: "Mul_204" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_205/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_205/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_205" op: "Mul" input: "Mul_205/x" input: "Mul_205/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_37" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_206/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_206" op: "Mul" input: "Mul_206/x" input: "Square_37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_128" op: "AddV2" input: "Mul_205" input: "Mul_206" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_37" op: "Sqrt" input: "add_128" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_129/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_129" op: "AddV2" input: "Sqrt_37" input: "add_129/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_38" op: "RealDiv" input: "add_127" input: "add_129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_163" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_207/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_207" op: "Mul" input: "mul_207/x" input: "ReadVariableOp_163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_130" op: "AddV2" input: "truediv_38" input: "mul_207" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_208" op: "Mul" input: "add" input: "add_130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_164" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_38" op: "Sub" input: "ReadVariableOp_164" input: "mul_208" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_260" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel" input: "sub_38" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_165" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel" input: "^AssignVariableOp_260" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_261" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" input: "add_127" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_166" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" input: "^AssignVariableOp_261" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_262" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" input: "add_128" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_167" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" input: "^AssignVariableOp_262" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/query/bias/adam_m" } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/query/bias/adam_v" } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/query/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_209/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_209/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_209" op: "Mul" input: "Mul_209/x" input: "Mul_209/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_210/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_210" op: "Mul" input: "Mul_210/x" input: "clip_by_global_norm/clip_by_global_norm/_38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_131" op: "AddV2" input: "Mul_209" input: "Mul_210" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_211/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_211/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_211" op: "Mul" input: "Mul_211/x" input: "Mul_211/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_38" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_212/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_212" op: "Mul" input: "Mul_212/x" input: "Square_38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_132" op: "AddV2" input: "Mul_211" input: "Mul_212" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_38" op: "Sqrt" input: "add_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_133/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_133" op: "AddV2" input: "Sqrt_38" input: "add_133/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_39" op: "RealDiv" input: "add_131" input: "add_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_213" op: "Mul" input: "add" input: "truediv_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_168" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_39" op: "Sub" input: "ReadVariableOp_168" input: "mul_213" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_263" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias" input: "sub_39" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_169" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias" input: "^AssignVariableOp_263" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_264" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m" input: "add_131" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_170" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m" input: "^AssignVariableOp_264" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_265" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v" input: "add_132" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_171" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v" input: "^AssignVariableOp_265" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_214/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_214/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_214" op: "Mul" input: "Mul_214/x" input: "Mul_214/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_215/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_215" op: "Mul" input: "Mul_215/x" input: "clip_by_global_norm/clip_by_global_norm/_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_134" op: "AddV2" input: "Mul_214" input: "Mul_215" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_216/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_216/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_216" op: "Mul" input: "Mul_216/x" input: "Mul_216/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_39" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_217/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_217" op: "Mul" input: "Mul_217/x" input: "Square_39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_135" op: "AddV2" input: "Mul_216" input: "Mul_217" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_39" op: "Sqrt" input: "add_135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_136/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_136" op: "AddV2" input: "Sqrt_39" input: "add_136/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_40" op: "RealDiv" input: "add_134" input: "add_136" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_172" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_218/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_218" op: "Mul" input: "mul_218/x" input: "ReadVariableOp_172" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_137" op: "AddV2" input: "truediv_40" input: "mul_218" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_219" op: "Mul" input: "add" input: "add_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_173" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_40" op: "Sub" input: "ReadVariableOp_173" input: "mul_219" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_266" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel" input: "sub_40" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_174" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel" input: "^AssignVariableOp_266" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_267" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" input: "add_134" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_175" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" input: "^AssignVariableOp_267" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_268" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" input: "add_135" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_176" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" input: "^AssignVariableOp_268" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/key/bias/adam_m" } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/key/bias/adam_v" } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/key/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_220/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_220/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_220" op: "Mul" input: "Mul_220/x" input: "Mul_220/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_221/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_221" op: "Mul" input: "Mul_221/x" input: "clip_by_global_norm/clip_by_global_norm/_40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_138" op: "AddV2" input: "Mul_220" input: "Mul_221" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_222/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_222/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_222" op: "Mul" input: "Mul_222/x" input: "Mul_222/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_40" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_223/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_223" op: "Mul" input: "Mul_223/x" input: "Square_40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_139" op: "AddV2" input: "Mul_222" input: "Mul_223" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_40" op: "Sqrt" input: "add_139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_140/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_140" op: "AddV2" input: "Sqrt_40" input: "add_140/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_41" op: "RealDiv" input: "add_138" input: "add_140" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_224" op: "Mul" input: "add" input: "truediv_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_177" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_41" op: "Sub" input: "ReadVariableOp_177" input: "mul_224" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_269" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias" input: "sub_41" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_178" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias" input: "^AssignVariableOp_269" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_270" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m" input: "add_138" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_179" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m" input: "^AssignVariableOp_270" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_271" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v" input: "add_139" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_180" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v" input: "^AssignVariableOp_271" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_225/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_225/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_225" op: "Mul" input: "Mul_225/x" input: "Mul_225/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_226/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_226" op: "Mul" input: "Mul_226/x" input: "clip_by_global_norm/clip_by_global_norm/_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_141" op: "AddV2" input: "Mul_225" input: "Mul_226" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_227/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_227/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_227" op: "Mul" input: "Mul_227/x" input: "Mul_227/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_41" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_228/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_228" op: "Mul" input: "Mul_228/x" input: "Square_41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_142" op: "AddV2" input: "Mul_227" input: "Mul_228" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_41" op: "Sqrt" input: "add_142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_143/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_143" op: "AddV2" input: "Sqrt_41" input: "add_143/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_42" op: "RealDiv" input: "add_141" input: "add_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_181" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_229/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_229" op: "Mul" input: "mul_229/x" input: "ReadVariableOp_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_144" op: "AddV2" input: "truediv_42" input: "mul_229" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_230" op: "Mul" input: "add" input: "add_144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_182" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_42" op: "Sub" input: "ReadVariableOp_182" input: "mul_230" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_272" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel" input: "sub_42" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_183" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel" input: "^AssignVariableOp_272" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_273" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" input: "add_141" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_184" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" input: "^AssignVariableOp_273" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_274" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" input: "add_142" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_185" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" input: "^AssignVariableOp_274" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/value/bias/adam_m" } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/self/value/bias/adam_v" } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/self/value/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_231/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_231/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_231" op: "Mul" input: "Mul_231/x" input: "Mul_231/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_232/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_232" op: "Mul" input: "Mul_232/x" input: "clip_by_global_norm/clip_by_global_norm/_42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_145" op: "AddV2" input: "Mul_231" input: "Mul_232" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_233/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_233/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_233" op: "Mul" input: "Mul_233/x" input: "Mul_233/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_42" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_234/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_234" op: "Mul" input: "Mul_234/x" input: "Square_42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_146" op: "AddV2" input: "Mul_233" input: "Mul_234" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_42" op: "Sqrt" input: "add_146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_147/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_147" op: "AddV2" input: "Sqrt_42" input: "add_147/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_43" op: "RealDiv" input: "add_145" input: "add_147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_235" op: "Mul" input: "add" input: "truediv_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_186" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_43" op: "Sub" input: "ReadVariableOp_186" input: "mul_235" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_275" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias" input: "sub_43" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_187" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias" input: "^AssignVariableOp_275" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_276" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m" input: "add_145" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_188" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m" input: "^AssignVariableOp_276" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_277" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v" input: "add_146" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_189" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v" input: "^AssignVariableOp_277" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_236/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_236/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_236" op: "Mul" input: "Mul_236/x" input: "Mul_236/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_237/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_237" op: "Mul" input: "Mul_237/x" input: "clip_by_global_norm/clip_by_global_norm/_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_148" op: "AddV2" input: "Mul_236" input: "Mul_237" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_238/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_238/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_238" op: "Mul" input: "Mul_238/x" input: "Mul_238/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_43" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_239/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_239" op: "Mul" input: "Mul_239/x" input: "Square_43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_149" op: "AddV2" input: "Mul_238" input: "Mul_239" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_43" op: "Sqrt" input: "add_149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_150/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_150" op: "AddV2" input: "Sqrt_43" input: "add_150/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_44" op: "RealDiv" input: "add_148" input: "add_150" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_190" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_240/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_240" op: "Mul" input: "mul_240/x" input: "ReadVariableOp_190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_151" op: "AddV2" input: "truediv_44" input: "mul_240" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_241" op: "Mul" input: "add" input: "add_151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_191" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_44" op: "Sub" input: "ReadVariableOp_191" input: "mul_241" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_278" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel" input: "sub_44" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_192" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel" input: "^AssignVariableOp_278" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_279" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" input: "add_148" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_193" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" input: "^AssignVariableOp_279" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_280" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" input: "add_149" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_194" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" input: "^AssignVariableOp_280" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_242/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_242/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_242" op: "Mul" input: "Mul_242/x" input: "Mul_242/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_243/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_243" op: "Mul" input: "Mul_243/x" input: "clip_by_global_norm/clip_by_global_norm/_44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_152" op: "AddV2" input: "Mul_242" input: "Mul_243" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_244/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_244/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_244" op: "Mul" input: "Mul_244/x" input: "Mul_244/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_44" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_245/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_245" op: "Mul" input: "Mul_245/x" input: "Square_44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_153" op: "AddV2" input: "Mul_244" input: "Mul_245" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_44" op: "Sqrt" input: "add_153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_154/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_154" op: "AddV2" input: "Sqrt_44" input: "add_154/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_45" op: "RealDiv" input: "add_152" input: "add_154" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_246" op: "Mul" input: "add" input: "truediv_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_195" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_45" op: "Sub" input: "ReadVariableOp_195" input: "mul_246" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_281" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias" input: "sub_45" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_196" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias" input: "^AssignVariableOp_281" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_282" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" input: "add_152" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_197" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" input: "^AssignVariableOp_282" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_283" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" input: "add_153" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_198" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" input: "^AssignVariableOp_283" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_247/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_247/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_247" op: "Mul" input: "Mul_247/x" input: "Mul_247/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_248/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_248" op: "Mul" input: "Mul_248/x" input: "clip_by_global_norm/clip_by_global_norm/_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_155" op: "AddV2" input: "Mul_247" input: "Mul_248" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_249/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_249/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_249" op: "Mul" input: "Mul_249/x" input: "Mul_249/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_45" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_250/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_250" op: "Mul" input: "Mul_250/x" input: "Square_45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_156" op: "AddV2" input: "Mul_249" input: "Mul_250" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_45" op: "Sqrt" input: "add_156" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_157/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_157" op: "AddV2" input: "Sqrt_45" input: "add_157/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_46" op: "RealDiv" input: "add_155" input: "add_157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_251" op: "Mul" input: "add" input: "truediv_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_199" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_46" op: "Sub" input: "ReadVariableOp_199" input: "mul_251" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_284" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" input: "sub_46" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_200" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" input: "^AssignVariableOp_284" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_285" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" input: "add_155" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_201" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" input: "^AssignVariableOp_285" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_286" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" input: "add_156" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_202" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" input: "^AssignVariableOp_286" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_252/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_252/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_252" op: "Mul" input: "Mul_252/x" input: "Mul_252/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_253/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_253" op: "Mul" input: "Mul_253/x" input: "clip_by_global_norm/clip_by_global_norm/_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_158" op: "AddV2" input: "Mul_252" input: "Mul_253" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_254/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_254/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_254" op: "Mul" input: "Mul_254/x" input: "Mul_254/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_46" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_255/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_255" op: "Mul" input: "Mul_255/x" input: "Square_46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_159" op: "AddV2" input: "Mul_254" input: "Mul_255" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_46" op: "Sqrt" input: "add_159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_160/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_160" op: "AddV2" input: "Sqrt_46" input: "add_160/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_47" op: "RealDiv" input: "add_158" input: "add_160" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_256" op: "Mul" input: "add" input: "truediv_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_203" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_47" op: "Sub" input: "ReadVariableOp_203" input: "mul_256" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_287" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta" input: "sub_47" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_204" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta" input: "^AssignVariableOp_287" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_288" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" input: "add_158" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_205" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" input: "^AssignVariableOp_288" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_289" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" input: "add_159" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_206" op: "ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" input: "^AssignVariableOp_289" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_257/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_257/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_257" op: "Mul" input: "Mul_257/x" input: "Mul_257/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_258/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_258" op: "Mul" input: "Mul_258/x" input: "clip_by_global_norm/clip_by_global_norm/_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_161" op: "AddV2" input: "Mul_257" input: "Mul_258" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_259/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_259/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_259" op: "Mul" input: "Mul_259/x" input: "Mul_259/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_47" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_260/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_260" op: "Mul" input: "Mul_260/x" input: "Square_47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_162" op: "AddV2" input: "Mul_259" input: "Mul_260" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_47" op: "Sqrt" input: "add_162" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_163/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_163" op: "AddV2" input: "Sqrt_47" input: "add_163/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_48" op: "RealDiv" input: "add_161" input: "add_163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_207" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_261/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_261" op: "Mul" input: "mul_261/x" input: "ReadVariableOp_207" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_164" op: "AddV2" input: "truediv_48" input: "mul_261" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_262" op: "Mul" input: "add" input: "add_164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_208" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_48" op: "Sub" input: "ReadVariableOp_208" input: "mul_262" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "AssignVariableOp_290" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel" input: "sub_48" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_209" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel" input: "^AssignVariableOp_290" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_291" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" input: "add_161" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_210" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" input: "^AssignVariableOp_291" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_292" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" input: "add_162" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_211" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" input: "^AssignVariableOp_292" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_263/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_263/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_263" op: "Mul" input: "Mul_263/x" input: "Mul_263/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_264/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_264" op: "Mul" input: "Mul_264/x" input: "clip_by_global_norm/clip_by_global_norm/_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_165" op: "AddV2" input: "Mul_263" input: "Mul_264" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_265/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_265/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_265" op: "Mul" input: "Mul_265/x" input: "Mul_265/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_48" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_266/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_266" op: "Mul" input: "Mul_266/x" input: "Square_48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_166" op: "AddV2" input: "Mul_265" input: "Mul_266" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_48" op: "Sqrt" input: "add_166" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_167/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_167" op: "AddV2" input: "Sqrt_48" input: "add_167/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_49" op: "RealDiv" input: "add_165" input: "add_167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_267" op: "Mul" input: "add" input: "truediv_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "ReadVariableOp_212" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_49" op: "Sub" input: "ReadVariableOp_212" input: "mul_267" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "AssignVariableOp_293" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias" input: "sub_49" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_213" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias" input: "^AssignVariableOp_293" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_294" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" input: "add_165" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_214" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" input: "^AssignVariableOp_294" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_295" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" input: "add_166" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_215" op: "ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" input: "^AssignVariableOp_295" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_m" input: "bert/encoder/layer_2/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_v" input: "bert/encoder/layer_2/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_268/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_268/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_268" op: "Mul" input: "Mul_268/x" input: "Mul_268/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_269/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_269" op: "Mul" input: "Mul_269/x" input: "clip_by_global_norm/clip_by_global_norm/_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_168" op: "AddV2" input: "Mul_268" input: "Mul_269" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_270/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_270/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_270" op: "Mul" input: "Mul_270/x" input: "Mul_270/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_49" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_271/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_271" op: "Mul" input: "Mul_271/x" input: "Square_49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_169" op: "AddV2" input: "Mul_270" input: "Mul_271" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_49" op: "Sqrt" input: "add_169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_170/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_170" op: "AddV2" input: "Sqrt_49" input: "add_170/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_50" op: "RealDiv" input: "add_168" input: "add_170" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_216" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_272/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_272" op: "Mul" input: "mul_272/x" input: "ReadVariableOp_216" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_171" op: "AddV2" input: "truediv_50" input: "mul_272" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_273" op: "Mul" input: "add" input: "add_171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_217" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_50" op: "Sub" input: "ReadVariableOp_217" input: "mul_273" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_296" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/kernel" input: "sub_50" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_218" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/kernel" input: "^AssignVariableOp_296" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_297" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_m" input: "add_168" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_219" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_m" input: "^AssignVariableOp_297" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_298" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_v" input: "add_169" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_220" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_v" input: "^AssignVariableOp_298" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/bias/adam_m" input: "bert/encoder/layer_2/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/bias/adam_v" input: "bert/encoder/layer_2/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_274/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_274/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_274" op: "Mul" input: "Mul_274/x" input: "Mul_274/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_275/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_275" op: "Mul" input: "Mul_275/x" input: "clip_by_global_norm/clip_by_global_norm/_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_172" op: "AddV2" input: "Mul_274" input: "Mul_275" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_276/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_276/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_276" op: "Mul" input: "Mul_276/x" input: "Mul_276/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_50" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_277/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_277" op: "Mul" input: "Mul_277/x" input: "Square_50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_173" op: "AddV2" input: "Mul_276" input: "Mul_277" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_50" op: "Sqrt" input: "add_173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_174/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_174" op: "AddV2" input: "Sqrt_50" input: "add_174/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_51" op: "RealDiv" input: "add_172" input: "add_174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_278" op: "Mul" input: "add" input: "truediv_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_221" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_51" op: "Sub" input: "ReadVariableOp_221" input: "mul_278" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_299" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/bias" input: "sub_51" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_222" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/bias" input: "^AssignVariableOp_299" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_300" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/bias/adam_m" input: "add_172" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_223" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/bias/adam_m" input: "^AssignVariableOp_300" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_301" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/bias/adam_v" input: "add_173" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_224" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/dense/bias/adam_v" input: "^AssignVariableOp_301" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_279/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_279/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_279" op: "Mul" input: "Mul_279/x" input: "Mul_279/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_280/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_280" op: "Mul" input: "Mul_280/x" input: "clip_by_global_norm/clip_by_global_norm/_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_175" op: "AddV2" input: "Mul_279" input: "Mul_280" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_281/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_281/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_281" op: "Mul" input: "Mul_281/x" input: "Mul_281/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_51" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_282/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_282" op: "Mul" input: "Mul_282/x" input: "Square_51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_176" op: "AddV2" input: "Mul_281" input: "Mul_282" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_51" op: "Sqrt" input: "add_176" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_177/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_177" op: "AddV2" input: "Sqrt_51" input: "add_177/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_52" op: "RealDiv" input: "add_175" input: "add_177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_283" op: "Mul" input: "add" input: "truediv_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_225" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_52" op: "Sub" input: "ReadVariableOp_225" input: "mul_283" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_302" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma" input: "sub_52" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_226" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma" input: "^AssignVariableOp_302" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_303" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" input: "add_175" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_227" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" input: "^AssignVariableOp_303" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_304" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" input: "add_176" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_228" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" input: "^AssignVariableOp_304" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_284/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_284/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_284" op: "Mul" input: "Mul_284/x" input: "Mul_284/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_285/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_285" op: "Mul" input: "Mul_285/x" input: "clip_by_global_norm/clip_by_global_norm/_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_178" op: "AddV2" input: "Mul_284" input: "Mul_285" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_286/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_286/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_286" op: "Mul" input: "Mul_286/x" input: "Mul_286/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_52" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_287/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_287" op: "Mul" input: "Mul_287/x" input: "Square_52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_179" op: "AddV2" input: "Mul_286" input: "Mul_287" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_52" op: "Sqrt" input: "add_179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_180/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_180" op: "AddV2" input: "Sqrt_52" input: "add_180/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_53" op: "RealDiv" input: "add_178" input: "add_180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_288" op: "Mul" input: "add" input: "truediv_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_229" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_53" op: "Sub" input: "ReadVariableOp_229" input: "mul_288" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_305" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta" input: "sub_53" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_230" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta" input: "^AssignVariableOp_305" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_306" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" input: "add_178" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_231" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" input: "^AssignVariableOp_306" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_307" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" input: "add_179" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_232" op: "ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" input: "^AssignVariableOp_307" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_289/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_289/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_289" op: "Mul" input: "Mul_289/x" input: "Mul_289/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_290/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_290" op: "Mul" input: "Mul_290/x" input: "clip_by_global_norm/clip_by_global_norm/_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_181" op: "AddV2" input: "Mul_289" input: "Mul_290" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_291/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_291/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_291" op: "Mul" input: "Mul_291/x" input: "Mul_291/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_53" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_292/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_292" op: "Mul" input: "Mul_292/x" input: "Square_53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_182" op: "AddV2" input: "Mul_291" input: "Mul_292" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_53" op: "Sqrt" input: "add_182" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_183/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_183" op: "AddV2" input: "Sqrt_53" input: "add_183/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_54" op: "RealDiv" input: "add_181" input: "add_183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_233" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_293/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_293" op: "Mul" input: "mul_293/x" input: "ReadVariableOp_233" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_184" op: "AddV2" input: "truediv_54" input: "mul_293" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_294" op: "Mul" input: "add" input: "add_184" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_234" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_54" op: "Sub" input: "ReadVariableOp_234" input: "mul_294" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_308" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel" input: "sub_54" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_235" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel" input: "^AssignVariableOp_308" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_309" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" input: "add_181" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_236" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" input: "^AssignVariableOp_309" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_310" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" input: "add_182" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_237" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" input: "^AssignVariableOp_310" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/query/bias/adam_m" } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/query/bias/adam_v" } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/query/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_295/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_295/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_295" op: "Mul" input: "Mul_295/x" input: "Mul_295/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_296/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_296" op: "Mul" input: "Mul_296/x" input: "clip_by_global_norm/clip_by_global_norm/_54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_185" op: "AddV2" input: "Mul_295" input: "Mul_296" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_297/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_297/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_297" op: "Mul" input: "Mul_297/x" input: "Mul_297/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_54" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_298/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_298" op: "Mul" input: "Mul_298/x" input: "Square_54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_186" op: "AddV2" input: "Mul_297" input: "Mul_298" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_54" op: "Sqrt" input: "add_186" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_187/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_187" op: "AddV2" input: "Sqrt_54" input: "add_187/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_55" op: "RealDiv" input: "add_185" input: "add_187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_299" op: "Mul" input: "add" input: "truediv_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_238" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_55" op: "Sub" input: "ReadVariableOp_238" input: "mul_299" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_311" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias" input: "sub_55" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_239" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias" input: "^AssignVariableOp_311" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_312" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m" input: "add_185" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_240" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m" input: "^AssignVariableOp_312" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_313" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v" input: "add_186" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_241" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v" input: "^AssignVariableOp_313" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_300/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_300/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_300" op: "Mul" input: "Mul_300/x" input: "Mul_300/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_301/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_301" op: "Mul" input: "Mul_301/x" input: "clip_by_global_norm/clip_by_global_norm/_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_188" op: "AddV2" input: "Mul_300" input: "Mul_301" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_302/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_302/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_302" op: "Mul" input: "Mul_302/x" input: "Mul_302/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_55" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_303/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_303" op: "Mul" input: "Mul_303/x" input: "Square_55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_189" op: "AddV2" input: "Mul_302" input: "Mul_303" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_55" op: "Sqrt" input: "add_189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_190/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_190" op: "AddV2" input: "Sqrt_55" input: "add_190/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_56" op: "RealDiv" input: "add_188" input: "add_190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_242" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_304/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_304" op: "Mul" input: "mul_304/x" input: "ReadVariableOp_242" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_191" op: "AddV2" input: "truediv_56" input: "mul_304" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_305" op: "Mul" input: "add" input: "add_191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_243" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_56" op: "Sub" input: "ReadVariableOp_243" input: "mul_305" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_314" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel" input: "sub_56" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_244" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel" input: "^AssignVariableOp_314" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_315" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" input: "add_188" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_245" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" input: "^AssignVariableOp_315" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_316" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" input: "add_189" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_246" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" input: "^AssignVariableOp_316" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/key/bias/adam_m" } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/key/bias/adam_v" } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/key/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_306/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_306/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_306" op: "Mul" input: "Mul_306/x" input: "Mul_306/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_307/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_307" op: "Mul" input: "Mul_307/x" input: "clip_by_global_norm/clip_by_global_norm/_56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_192" op: "AddV2" input: "Mul_306" input: "Mul_307" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_308/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_308/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_308" op: "Mul" input: "Mul_308/x" input: "Mul_308/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_56" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_309/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_309" op: "Mul" input: "Mul_309/x" input: "Square_56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_193" op: "AddV2" input: "Mul_308" input: "Mul_309" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_56" op: "Sqrt" input: "add_193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_194/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_194" op: "AddV2" input: "Sqrt_56" input: "add_194/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_57" op: "RealDiv" input: "add_192" input: "add_194" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_310" op: "Mul" input: "add" input: "truediv_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_247" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_57" op: "Sub" input: "ReadVariableOp_247" input: "mul_310" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_317" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias" input: "sub_57" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_248" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias" input: "^AssignVariableOp_317" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_318" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m" input: "add_192" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_249" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m" input: "^AssignVariableOp_318" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_319" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v" input: "add_193" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_250" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v" input: "^AssignVariableOp_319" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_311/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_311/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_311" op: "Mul" input: "Mul_311/x" input: "Mul_311/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_312/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_312" op: "Mul" input: "Mul_312/x" input: "clip_by_global_norm/clip_by_global_norm/_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_195" op: "AddV2" input: "Mul_311" input: "Mul_312" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_313/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_313/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_313" op: "Mul" input: "Mul_313/x" input: "Mul_313/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_57" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_314/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_314" op: "Mul" input: "Mul_314/x" input: "Square_57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_196" op: "AddV2" input: "Mul_313" input: "Mul_314" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_57" op: "Sqrt" input: "add_196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_197/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_197" op: "AddV2" input: "Sqrt_57" input: "add_197/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_58" op: "RealDiv" input: "add_195" input: "add_197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_251" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_315/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_315" op: "Mul" input: "mul_315/x" input: "ReadVariableOp_251" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_198" op: "AddV2" input: "truediv_58" input: "mul_315" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_316" op: "Mul" input: "add" input: "add_198" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_252" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_58" op: "Sub" input: "ReadVariableOp_252" input: "mul_316" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_320" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel" input: "sub_58" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_253" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel" input: "^AssignVariableOp_320" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_321" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" input: "add_195" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_254" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" input: "^AssignVariableOp_321" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_322" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" input: "add_196" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_255" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" input: "^AssignVariableOp_322" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/value/bias/adam_m" } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/self/value/bias/adam_v" } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/self/value/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_317/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_317/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_317" op: "Mul" input: "Mul_317/x" input: "Mul_317/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_318/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_318" op: "Mul" input: "Mul_318/x" input: "clip_by_global_norm/clip_by_global_norm/_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_199" op: "AddV2" input: "Mul_317" input: "Mul_318" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_319/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_319/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_319" op: "Mul" input: "Mul_319/x" input: "Mul_319/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_58" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_320/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_320" op: "Mul" input: "Mul_320/x" input: "Square_58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_200" op: "AddV2" input: "Mul_319" input: "Mul_320" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_58" op: "Sqrt" input: "add_200" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_201/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_201" op: "AddV2" input: "Sqrt_58" input: "add_201/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_59" op: "RealDiv" input: "add_199" input: "add_201" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_321" op: "Mul" input: "add" input: "truediv_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_256" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_59" op: "Sub" input: "ReadVariableOp_256" input: "mul_321" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_323" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias" input: "sub_59" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_257" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias" input: "^AssignVariableOp_323" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_324" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m" input: "add_199" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_258" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m" input: "^AssignVariableOp_324" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_325" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v" input: "add_200" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_259" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v" input: "^AssignVariableOp_325" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_322/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_322/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_322" op: "Mul" input: "Mul_322/x" input: "Mul_322/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_323/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_323" op: "Mul" input: "Mul_323/x" input: "clip_by_global_norm/clip_by_global_norm/_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_202" op: "AddV2" input: "Mul_322" input: "Mul_323" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_324/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_324/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_324" op: "Mul" input: "Mul_324/x" input: "Mul_324/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_59" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_325/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_325" op: "Mul" input: "Mul_325/x" input: "Square_59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_203" op: "AddV2" input: "Mul_324" input: "Mul_325" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_59" op: "Sqrt" input: "add_203" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_204/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_204" op: "AddV2" input: "Sqrt_59" input: "add_204/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_60" op: "RealDiv" input: "add_202" input: "add_204" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_260" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_326/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_326" op: "Mul" input: "mul_326/x" input: "ReadVariableOp_260" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_205" op: "AddV2" input: "truediv_60" input: "mul_326" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_327" op: "Mul" input: "add" input: "add_205" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_261" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_60" op: "Sub" input: "ReadVariableOp_261" input: "mul_327" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_326" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel" input: "sub_60" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_262" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel" input: "^AssignVariableOp_326" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_327" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" input: "add_202" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_263" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" input: "^AssignVariableOp_327" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_328" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" input: "add_203" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_264" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" input: "^AssignVariableOp_328" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_328/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_328/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_328" op: "Mul" input: "Mul_328/x" input: "Mul_328/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_329/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_329" op: "Mul" input: "Mul_329/x" input: "clip_by_global_norm/clip_by_global_norm/_60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_206" op: "AddV2" input: "Mul_328" input: "Mul_329" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_330/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_330/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_330" op: "Mul" input: "Mul_330/x" input: "Mul_330/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_60" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_331/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_331" op: "Mul" input: "Mul_331/x" input: "Square_60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_207" op: "AddV2" input: "Mul_330" input: "Mul_331" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_60" op: "Sqrt" input: "add_207" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_208/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_208" op: "AddV2" input: "Sqrt_60" input: "add_208/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_61" op: "RealDiv" input: "add_206" input: "add_208" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_332" op: "Mul" input: "add" input: "truediv_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_265" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_61" op: "Sub" input: "ReadVariableOp_265" input: "mul_332" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_329" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias" input: "sub_61" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_266" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias" input: "^AssignVariableOp_329" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_330" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" input: "add_206" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_267" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" input: "^AssignVariableOp_330" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_331" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" input: "add_207" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_268" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" input: "^AssignVariableOp_331" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_333/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_333/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_333" op: "Mul" input: "Mul_333/x" input: "Mul_333/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_334/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_334" op: "Mul" input: "Mul_334/x" input: "clip_by_global_norm/clip_by_global_norm/_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_209" op: "AddV2" input: "Mul_333" input: "Mul_334" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_335/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_335/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_335" op: "Mul" input: "Mul_335/x" input: "Mul_335/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_61" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_336/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_336" op: "Mul" input: "Mul_336/x" input: "Square_61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_210" op: "AddV2" input: "Mul_335" input: "Mul_336" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_61" op: "Sqrt" input: "add_210" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_211/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_211" op: "AddV2" input: "Sqrt_61" input: "add_211/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_62" op: "RealDiv" input: "add_209" input: "add_211" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_337" op: "Mul" input: "add" input: "truediv_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_269" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_62" op: "Sub" input: "ReadVariableOp_269" input: "mul_337" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_332" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" input: "sub_62" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_270" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" input: "^AssignVariableOp_332" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_333" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" input: "add_209" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_271" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" input: "^AssignVariableOp_333" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_334" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" input: "add_210" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_272" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" input: "^AssignVariableOp_334" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_338/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_338/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_338" op: "Mul" input: "Mul_338/x" input: "Mul_338/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_339/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_339" op: "Mul" input: "Mul_339/x" input: "clip_by_global_norm/clip_by_global_norm/_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_212" op: "AddV2" input: "Mul_338" input: "Mul_339" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_340/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_340/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_340" op: "Mul" input: "Mul_340/x" input: "Mul_340/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_62" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_341/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_341" op: "Mul" input: "Mul_341/x" input: "Square_62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_213" op: "AddV2" input: "Mul_340" input: "Mul_341" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_62" op: "Sqrt" input: "add_213" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_214/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_214" op: "AddV2" input: "Sqrt_62" input: "add_214/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_63" op: "RealDiv" input: "add_212" input: "add_214" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_342" op: "Mul" input: "add" input: "truediv_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_273" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_63" op: "Sub" input: "ReadVariableOp_273" input: "mul_342" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_335" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta" input: "sub_63" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_274" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta" input: "^AssignVariableOp_335" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_336" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" input: "add_212" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_275" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" input: "^AssignVariableOp_336" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_337" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" input: "add_213" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_276" op: "ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" input: "^AssignVariableOp_337" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_343/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_343/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_343" op: "Mul" input: "Mul_343/x" input: "Mul_343/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_344/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_344" op: "Mul" input: "Mul_344/x" input: "clip_by_global_norm/clip_by_global_norm/_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_215" op: "AddV2" input: "Mul_343" input: "Mul_344" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_345/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_345/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_345" op: "Mul" input: "Mul_345/x" input: "Mul_345/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_63" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_346/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_346" op: "Mul" input: "Mul_346/x" input: "Square_63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_216" op: "AddV2" input: "Mul_345" input: "Mul_346" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_63" op: "Sqrt" input: "add_216" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_217/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_217" op: "AddV2" input: "Sqrt_63" input: "add_217/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_64" op: "RealDiv" input: "add_215" input: "add_217" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_277" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_347/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_347" op: "Mul" input: "mul_347/x" input: "ReadVariableOp_277" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_218" op: "AddV2" input: "truediv_64" input: "mul_347" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_348" op: "Mul" input: "add" input: "add_218" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_278" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_64" op: "Sub" input: "ReadVariableOp_278" input: "mul_348" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "AssignVariableOp_338" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel" input: "sub_64" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_279" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel" input: "^AssignVariableOp_338" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_339" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" input: "add_215" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_280" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" input: "^AssignVariableOp_339" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_340" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" input: "add_216" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_281" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" input: "^AssignVariableOp_340" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_349/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_349/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_349" op: "Mul" input: "Mul_349/x" input: "Mul_349/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_350/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_350" op: "Mul" input: "Mul_350/x" input: "clip_by_global_norm/clip_by_global_norm/_64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_219" op: "AddV2" input: "Mul_349" input: "Mul_350" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_351/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_351/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_351" op: "Mul" input: "Mul_351/x" input: "Mul_351/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_64" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_352/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_352" op: "Mul" input: "Mul_352/x" input: "Square_64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_220" op: "AddV2" input: "Mul_351" input: "Mul_352" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_64" op: "Sqrt" input: "add_220" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_221/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_221" op: "AddV2" input: "Sqrt_64" input: "add_221/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_65" op: "RealDiv" input: "add_219" input: "add_221" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_353" op: "Mul" input: "add" input: "truediv_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "ReadVariableOp_282" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_65" op: "Sub" input: "ReadVariableOp_282" input: "mul_353" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "AssignVariableOp_341" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias" input: "sub_65" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_283" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias" input: "^AssignVariableOp_341" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_342" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" input: "add_219" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_284" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" input: "^AssignVariableOp_342" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_343" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" input: "add_220" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_285" op: "ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" input: "^AssignVariableOp_343" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_m" input: "bert/encoder/layer_3/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_v" input: "bert/encoder/layer_3/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_354/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_354/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_354" op: "Mul" input: "Mul_354/x" input: "Mul_354/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_355/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_355" op: "Mul" input: "Mul_355/x" input: "clip_by_global_norm/clip_by_global_norm/_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_222" op: "AddV2" input: "Mul_354" input: "Mul_355" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_356/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_356/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_356" op: "Mul" input: "Mul_356/x" input: "Mul_356/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_65" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_357/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_357" op: "Mul" input: "Mul_357/x" input: "Square_65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_223" op: "AddV2" input: "Mul_356" input: "Mul_357" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_65" op: "Sqrt" input: "add_223" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_224/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_224" op: "AddV2" input: "Sqrt_65" input: "add_224/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_66" op: "RealDiv" input: "add_222" input: "add_224" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_286" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_358/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_358" op: "Mul" input: "mul_358/x" input: "ReadVariableOp_286" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_225" op: "AddV2" input: "truediv_66" input: "mul_358" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_359" op: "Mul" input: "add" input: "add_225" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_287" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_66" op: "Sub" input: "ReadVariableOp_287" input: "mul_359" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_344" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/kernel" input: "sub_66" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_288" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/kernel" input: "^AssignVariableOp_344" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_345" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_m" input: "add_222" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_289" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_m" input: "^AssignVariableOp_345" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_346" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_v" input: "add_223" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_290" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_v" input: "^AssignVariableOp_346" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/bias/adam_m" input: "bert/encoder/layer_3/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/bias/adam_v" input: "bert/encoder/layer_3/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_360/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_360/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_360" op: "Mul" input: "Mul_360/x" input: "Mul_360/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_361/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_361" op: "Mul" input: "Mul_361/x" input: "clip_by_global_norm/clip_by_global_norm/_66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_226" op: "AddV2" input: "Mul_360" input: "Mul_361" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_362/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_362/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_362" op: "Mul" input: "Mul_362/x" input: "Mul_362/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_66" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_363/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_363" op: "Mul" input: "Mul_363/x" input: "Square_66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_227" op: "AddV2" input: "Mul_362" input: "Mul_363" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_66" op: "Sqrt" input: "add_227" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_228/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_228" op: "AddV2" input: "Sqrt_66" input: "add_228/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_67" op: "RealDiv" input: "add_226" input: "add_228" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_364" op: "Mul" input: "add" input: "truediv_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_291" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_67" op: "Sub" input: "ReadVariableOp_291" input: "mul_364" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_347" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/bias" input: "sub_67" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_292" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/bias" input: "^AssignVariableOp_347" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_348" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/bias/adam_m" input: "add_226" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_293" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/bias/adam_m" input: "^AssignVariableOp_348" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_349" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/bias/adam_v" input: "add_227" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_294" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/dense/bias/adam_v" input: "^AssignVariableOp_349" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_365/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_365/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_365" op: "Mul" input: "Mul_365/x" input: "Mul_365/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_366/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_366" op: "Mul" input: "Mul_366/x" input: "clip_by_global_norm/clip_by_global_norm/_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_229" op: "AddV2" input: "Mul_365" input: "Mul_366" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_367/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_367/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_367" op: "Mul" input: "Mul_367/x" input: "Mul_367/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_67" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_368/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_368" op: "Mul" input: "Mul_368/x" input: "Square_67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_230" op: "AddV2" input: "Mul_367" input: "Mul_368" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_67" op: "Sqrt" input: "add_230" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_231/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_231" op: "AddV2" input: "Sqrt_67" input: "add_231/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_68" op: "RealDiv" input: "add_229" input: "add_231" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_369" op: "Mul" input: "add" input: "truediv_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_295" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_68" op: "Sub" input: "ReadVariableOp_295" input: "mul_369" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_350" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma" input: "sub_68" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_296" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma" input: "^AssignVariableOp_350" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_351" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" input: "add_229" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_297" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" input: "^AssignVariableOp_351" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_352" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" input: "add_230" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_298" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" input: "^AssignVariableOp_352" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_370/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_370/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_370" op: "Mul" input: "Mul_370/x" input: "Mul_370/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_371/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_371" op: "Mul" input: "Mul_371/x" input: "clip_by_global_norm/clip_by_global_norm/_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_232" op: "AddV2" input: "Mul_370" input: "Mul_371" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_372/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_372/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_372" op: "Mul" input: "Mul_372/x" input: "Mul_372/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_68" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_373/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_373" op: "Mul" input: "Mul_373/x" input: "Square_68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_233" op: "AddV2" input: "Mul_372" input: "Mul_373" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_68" op: "Sqrt" input: "add_233" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_234/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_234" op: "AddV2" input: "Sqrt_68" input: "add_234/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_69" op: "RealDiv" input: "add_232" input: "add_234" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_374" op: "Mul" input: "add" input: "truediv_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_299" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_69" op: "Sub" input: "ReadVariableOp_299" input: "mul_374" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_353" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta" input: "sub_69" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_300" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta" input: "^AssignVariableOp_353" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_354" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" input: "add_232" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_301" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" input: "^AssignVariableOp_354" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_355" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" input: "add_233" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_302" op: "ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" input: "^AssignVariableOp_355" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_375/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_375/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_375" op: "Mul" input: "Mul_375/x" input: "Mul_375/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_376/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_376" op: "Mul" input: "Mul_376/x" input: "clip_by_global_norm/clip_by_global_norm/_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_235" op: "AddV2" input: "Mul_375" input: "Mul_376" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_377/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_377/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_377" op: "Mul" input: "Mul_377/x" input: "Mul_377/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_69" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_378/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_378" op: "Mul" input: "Mul_378/x" input: "Square_69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_236" op: "AddV2" input: "Mul_377" input: "Mul_378" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_69" op: "Sqrt" input: "add_236" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_237/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_237" op: "AddV2" input: "Sqrt_69" input: "add_237/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_70" op: "RealDiv" input: "add_235" input: "add_237" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_303" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_379/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_379" op: "Mul" input: "mul_379/x" input: "ReadVariableOp_303" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_238" op: "AddV2" input: "truediv_70" input: "mul_379" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_380" op: "Mul" input: "add" input: "add_238" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_304" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_70" op: "Sub" input: "ReadVariableOp_304" input: "mul_380" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_356" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel" input: "sub_70" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_305" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel" input: "^AssignVariableOp_356" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_357" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" input: "add_235" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_306" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" input: "^AssignVariableOp_357" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_358" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" input: "add_236" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_307" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" input: "^AssignVariableOp_358" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/query/bias/adam_m" } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/query/bias/adam_v" } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/query/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_381/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_381/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_381" op: "Mul" input: "Mul_381/x" input: "Mul_381/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_382/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_382" op: "Mul" input: "Mul_382/x" input: "clip_by_global_norm/clip_by_global_norm/_70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_239" op: "AddV2" input: "Mul_381" input: "Mul_382" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_383/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_383/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_383" op: "Mul" input: "Mul_383/x" input: "Mul_383/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_70" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_384/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_384" op: "Mul" input: "Mul_384/x" input: "Square_70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_240" op: "AddV2" input: "Mul_383" input: "Mul_384" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_70" op: "Sqrt" input: "add_240" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_241/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_241" op: "AddV2" input: "Sqrt_70" input: "add_241/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_71" op: "RealDiv" input: "add_239" input: "add_241" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_385" op: "Mul" input: "add" input: "truediv_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_308" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_71" op: "Sub" input: "ReadVariableOp_308" input: "mul_385" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_359" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias" input: "sub_71" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_309" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias" input: "^AssignVariableOp_359" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_360" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m" input: "add_239" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_310" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m" input: "^AssignVariableOp_360" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_361" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v" input: "add_240" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_311" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v" input: "^AssignVariableOp_361" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_386/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_386/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_386" op: "Mul" input: "Mul_386/x" input: "Mul_386/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_387/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_387" op: "Mul" input: "Mul_387/x" input: "clip_by_global_norm/clip_by_global_norm/_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_242" op: "AddV2" input: "Mul_386" input: "Mul_387" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_388/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_388/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_388" op: "Mul" input: "Mul_388/x" input: "Mul_388/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_71" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_389/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_389" op: "Mul" input: "Mul_389/x" input: "Square_71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_243" op: "AddV2" input: "Mul_388" input: "Mul_389" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_71" op: "Sqrt" input: "add_243" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_244/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_244" op: "AddV2" input: "Sqrt_71" input: "add_244/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_72" op: "RealDiv" input: "add_242" input: "add_244" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_312" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_390/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_390" op: "Mul" input: "mul_390/x" input: "ReadVariableOp_312" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_245" op: "AddV2" input: "truediv_72" input: "mul_390" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_391" op: "Mul" input: "add" input: "add_245" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_313" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_72" op: "Sub" input: "ReadVariableOp_313" input: "mul_391" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_362" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel" input: "sub_72" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_314" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel" input: "^AssignVariableOp_362" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_363" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" input: "add_242" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_315" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" input: "^AssignVariableOp_363" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_364" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" input: "add_243" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_316" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" input: "^AssignVariableOp_364" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/key/bias/adam_m" } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/key/bias/adam_v" } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/key/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_392/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_392/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_392" op: "Mul" input: "Mul_392/x" input: "Mul_392/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_393/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_393" op: "Mul" input: "Mul_393/x" input: "clip_by_global_norm/clip_by_global_norm/_72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_246" op: "AddV2" input: "Mul_392" input: "Mul_393" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_394/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_394/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_394" op: "Mul" input: "Mul_394/x" input: "Mul_394/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_72" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_395/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_395" op: "Mul" input: "Mul_395/x" input: "Square_72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_247" op: "AddV2" input: "Mul_394" input: "Mul_395" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_72" op: "Sqrt" input: "add_247" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_248/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_248" op: "AddV2" input: "Sqrt_72" input: "add_248/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_73" op: "RealDiv" input: "add_246" input: "add_248" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_396" op: "Mul" input: "add" input: "truediv_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_317" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_73" op: "Sub" input: "ReadVariableOp_317" input: "mul_396" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_365" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias" input: "sub_73" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_318" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias" input: "^AssignVariableOp_365" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_366" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m" input: "add_246" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_319" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m" input: "^AssignVariableOp_366" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_367" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v" input: "add_247" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_320" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v" input: "^AssignVariableOp_367" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_397/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_397/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_397" op: "Mul" input: "Mul_397/x" input: "Mul_397/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_398/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_398" op: "Mul" input: "Mul_398/x" input: "clip_by_global_norm/clip_by_global_norm/_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_249" op: "AddV2" input: "Mul_397" input: "Mul_398" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_399/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_399/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_399" op: "Mul" input: "Mul_399/x" input: "Mul_399/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_73" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_400/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_400" op: "Mul" input: "Mul_400/x" input: "Square_73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_250" op: "AddV2" input: "Mul_399" input: "Mul_400" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_73" op: "Sqrt" input: "add_250" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_251/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_251" op: "AddV2" input: "Sqrt_73" input: "add_251/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_74" op: "RealDiv" input: "add_249" input: "add_251" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_321" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_401/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_401" op: "Mul" input: "mul_401/x" input: "ReadVariableOp_321" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_252" op: "AddV2" input: "truediv_74" input: "mul_401" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_402" op: "Mul" input: "add" input: "add_252" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_322" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_74" op: "Sub" input: "ReadVariableOp_322" input: "mul_402" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_368" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel" input: "sub_74" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_323" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel" input: "^AssignVariableOp_368" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_369" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" input: "add_249" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_324" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" input: "^AssignVariableOp_369" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_370" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" input: "add_250" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_325" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" input: "^AssignVariableOp_370" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/value/bias/adam_m" } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/self/value/bias/adam_v" } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/self/value/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_403/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_403/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_403" op: "Mul" input: "Mul_403/x" input: "Mul_403/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_404/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_404" op: "Mul" input: "Mul_404/x" input: "clip_by_global_norm/clip_by_global_norm/_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_253" op: "AddV2" input: "Mul_403" input: "Mul_404" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_405/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_405/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_405" op: "Mul" input: "Mul_405/x" input: "Mul_405/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_74" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_406/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_406" op: "Mul" input: "Mul_406/x" input: "Square_74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_254" op: "AddV2" input: "Mul_405" input: "Mul_406" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_74" op: "Sqrt" input: "add_254" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_255/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_255" op: "AddV2" input: "Sqrt_74" input: "add_255/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_75" op: "RealDiv" input: "add_253" input: "add_255" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_407" op: "Mul" input: "add" input: "truediv_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_326" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_75" op: "Sub" input: "ReadVariableOp_326" input: "mul_407" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_371" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias" input: "sub_75" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_327" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias" input: "^AssignVariableOp_371" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_372" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m" input: "add_253" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_328" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m" input: "^AssignVariableOp_372" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_373" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v" input: "add_254" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_329" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v" input: "^AssignVariableOp_373" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_408/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_408/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_408" op: "Mul" input: "Mul_408/x" input: "Mul_408/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_409/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_409" op: "Mul" input: "Mul_409/x" input: "clip_by_global_norm/clip_by_global_norm/_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_256" op: "AddV2" input: "Mul_408" input: "Mul_409" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_410/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_410/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_410" op: "Mul" input: "Mul_410/x" input: "Mul_410/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_75" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_411/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_411" op: "Mul" input: "Mul_411/x" input: "Square_75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_257" op: "AddV2" input: "Mul_410" input: "Mul_411" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_75" op: "Sqrt" input: "add_257" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_258/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_258" op: "AddV2" input: "Sqrt_75" input: "add_258/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_76" op: "RealDiv" input: "add_256" input: "add_258" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_330" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_412/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_412" op: "Mul" input: "mul_412/x" input: "ReadVariableOp_330" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_259" op: "AddV2" input: "truediv_76" input: "mul_412" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_413" op: "Mul" input: "add" input: "add_259" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_331" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_76" op: "Sub" input: "ReadVariableOp_331" input: "mul_413" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_374" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel" input: "sub_76" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_332" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel" input: "^AssignVariableOp_374" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_375" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" input: "add_256" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_333" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" input: "^AssignVariableOp_375" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_376" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" input: "add_257" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_334" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" input: "^AssignVariableOp_376" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_414/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_414/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_414" op: "Mul" input: "Mul_414/x" input: "Mul_414/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_415/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_415" op: "Mul" input: "Mul_415/x" input: "clip_by_global_norm/clip_by_global_norm/_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_260" op: "AddV2" input: "Mul_414" input: "Mul_415" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_416/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_416/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_416" op: "Mul" input: "Mul_416/x" input: "Mul_416/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_76" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_417/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_417" op: "Mul" input: "Mul_417/x" input: "Square_76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_261" op: "AddV2" input: "Mul_416" input: "Mul_417" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_76" op: "Sqrt" input: "add_261" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_262/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_262" op: "AddV2" input: "Sqrt_76" input: "add_262/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_77" op: "RealDiv" input: "add_260" input: "add_262" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_418" op: "Mul" input: "add" input: "truediv_77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_335" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_77" op: "Sub" input: "ReadVariableOp_335" input: "mul_418" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_377" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias" input: "sub_77" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_336" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias" input: "^AssignVariableOp_377" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_378" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" input: "add_260" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_337" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" input: "^AssignVariableOp_378" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_379" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" input: "add_261" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_338" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" input: "^AssignVariableOp_379" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_419/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_419/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_419" op: "Mul" input: "Mul_419/x" input: "Mul_419/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_420/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_420" op: "Mul" input: "Mul_420/x" input: "clip_by_global_norm/clip_by_global_norm/_77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_263" op: "AddV2" input: "Mul_419" input: "Mul_420" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_421/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_421/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_421" op: "Mul" input: "Mul_421/x" input: "Mul_421/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_77" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_422/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_422" op: "Mul" input: "Mul_422/x" input: "Square_77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_264" op: "AddV2" input: "Mul_421" input: "Mul_422" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_77" op: "Sqrt" input: "add_264" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_265/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_265" op: "AddV2" input: "Sqrt_77" input: "add_265/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_78" op: "RealDiv" input: "add_263" input: "add_265" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_423" op: "Mul" input: "add" input: "truediv_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_339" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_78" op: "Sub" input: "ReadVariableOp_339" input: "mul_423" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_380" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" input: "sub_78" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_340" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" input: "^AssignVariableOp_380" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_381" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" input: "add_263" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_341" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" input: "^AssignVariableOp_381" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_382" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" input: "add_264" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_342" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" input: "^AssignVariableOp_382" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_424/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_424/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_424" op: "Mul" input: "Mul_424/x" input: "Mul_424/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_425/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_425" op: "Mul" input: "Mul_425/x" input: "clip_by_global_norm/clip_by_global_norm/_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_266" op: "AddV2" input: "Mul_424" input: "Mul_425" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_426/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_426/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_426" op: "Mul" input: "Mul_426/x" input: "Mul_426/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_78" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_427/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_427" op: "Mul" input: "Mul_427/x" input: "Square_78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_267" op: "AddV2" input: "Mul_426" input: "Mul_427" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_78" op: "Sqrt" input: "add_267" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_268/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_268" op: "AddV2" input: "Sqrt_78" input: "add_268/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_79" op: "RealDiv" input: "add_266" input: "add_268" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_428" op: "Mul" input: "add" input: "truediv_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_343" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_79" op: "Sub" input: "ReadVariableOp_343" input: "mul_428" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_383" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta" input: "sub_79" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_344" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta" input: "^AssignVariableOp_383" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_384" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" input: "add_266" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_345" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" input: "^AssignVariableOp_384" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_385" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" input: "add_267" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_346" op: "ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" input: "^AssignVariableOp_385" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_429/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_429/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_429" op: "Mul" input: "Mul_429/x" input: "Mul_429/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_430/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_430" op: "Mul" input: "Mul_430/x" input: "clip_by_global_norm/clip_by_global_norm/_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_269" op: "AddV2" input: "Mul_429" input: "Mul_430" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_431/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_431/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_431" op: "Mul" input: "Mul_431/x" input: "Mul_431/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_79" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_432/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_432" op: "Mul" input: "Mul_432/x" input: "Square_79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_270" op: "AddV2" input: "Mul_431" input: "Mul_432" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_79" op: "Sqrt" input: "add_270" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_271/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_271" op: "AddV2" input: "Sqrt_79" input: "add_271/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_80" op: "RealDiv" input: "add_269" input: "add_271" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_347" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_433/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_433" op: "Mul" input: "mul_433/x" input: "ReadVariableOp_347" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_272" op: "AddV2" input: "truediv_80" input: "mul_433" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_434" op: "Mul" input: "add" input: "add_272" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_348" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_80" op: "Sub" input: "ReadVariableOp_348" input: "mul_434" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "AssignVariableOp_386" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel" input: "sub_80" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_349" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel" input: "^AssignVariableOp_386" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_387" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" input: "add_269" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_350" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" input: "^AssignVariableOp_387" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_388" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" input: "add_270" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_351" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" input: "^AssignVariableOp_388" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_435/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_435/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_435" op: "Mul" input: "Mul_435/x" input: "Mul_435/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_436/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_436" op: "Mul" input: "Mul_436/x" input: "clip_by_global_norm/clip_by_global_norm/_80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_273" op: "AddV2" input: "Mul_435" input: "Mul_436" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_437/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_437/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_437" op: "Mul" input: "Mul_437/x" input: "Mul_437/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_80" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_438/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_438" op: "Mul" input: "Mul_438/x" input: "Square_80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_274" op: "AddV2" input: "Mul_437" input: "Mul_438" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_80" op: "Sqrt" input: "add_274" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_275/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_275" op: "AddV2" input: "Sqrt_80" input: "add_275/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_81" op: "RealDiv" input: "add_273" input: "add_275" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_439" op: "Mul" input: "add" input: "truediv_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "ReadVariableOp_352" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_81" op: "Sub" input: "ReadVariableOp_352" input: "mul_439" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "AssignVariableOp_389" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias" input: "sub_81" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_353" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias" input: "^AssignVariableOp_389" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_390" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" input: "add_273" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_354" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" input: "^AssignVariableOp_390" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_391" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" input: "add_274" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_355" op: "ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" input: "^AssignVariableOp_391" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_m" input: "bert/encoder/layer_4/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_v" input: "bert/encoder/layer_4/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_440/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_440/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_440" op: "Mul" input: "Mul_440/x" input: "Mul_440/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_441/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_441" op: "Mul" input: "Mul_441/x" input: "clip_by_global_norm/clip_by_global_norm/_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_276" op: "AddV2" input: "Mul_440" input: "Mul_441" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_442/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_442/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_442" op: "Mul" input: "Mul_442/x" input: "Mul_442/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_81" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_443/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_443" op: "Mul" input: "Mul_443/x" input: "Square_81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_277" op: "AddV2" input: "Mul_442" input: "Mul_443" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_81" op: "Sqrt" input: "add_277" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_278/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_278" op: "AddV2" input: "Sqrt_81" input: "add_278/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_82" op: "RealDiv" input: "add_276" input: "add_278" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_356" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_444/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_444" op: "Mul" input: "mul_444/x" input: "ReadVariableOp_356" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_279" op: "AddV2" input: "truediv_82" input: "mul_444" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_445" op: "Mul" input: "add" input: "add_279" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_357" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_82" op: "Sub" input: "ReadVariableOp_357" input: "mul_445" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_392" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/kernel" input: "sub_82" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_358" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/kernel" input: "^AssignVariableOp_392" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_393" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_m" input: "add_276" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_359" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_m" input: "^AssignVariableOp_393" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_394" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_v" input: "add_277" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_360" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_v" input: "^AssignVariableOp_394" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/bias/adam_m" input: "bert/encoder/layer_4/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/bias/adam_v" input: "bert/encoder/layer_4/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_446/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_446/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_446" op: "Mul" input: "Mul_446/x" input: "Mul_446/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_447/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_447" op: "Mul" input: "Mul_447/x" input: "clip_by_global_norm/clip_by_global_norm/_82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_280" op: "AddV2" input: "Mul_446" input: "Mul_447" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_448/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_448/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_448" op: "Mul" input: "Mul_448/x" input: "Mul_448/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_82" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_449/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_449" op: "Mul" input: "Mul_449/x" input: "Square_82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_281" op: "AddV2" input: "Mul_448" input: "Mul_449" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_82" op: "Sqrt" input: "add_281" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_282/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_282" op: "AddV2" input: "Sqrt_82" input: "add_282/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_83" op: "RealDiv" input: "add_280" input: "add_282" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_450" op: "Mul" input: "add" input: "truediv_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_361" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_83" op: "Sub" input: "ReadVariableOp_361" input: "mul_450" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_395" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/bias" input: "sub_83" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_362" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/bias" input: "^AssignVariableOp_395" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_396" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/bias/adam_m" input: "add_280" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_363" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/bias/adam_m" input: "^AssignVariableOp_396" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_397" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/bias/adam_v" input: "add_281" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_364" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/dense/bias/adam_v" input: "^AssignVariableOp_397" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_451/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_451/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_451" op: "Mul" input: "Mul_451/x" input: "Mul_451/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_452/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_452" op: "Mul" input: "Mul_452/x" input: "clip_by_global_norm/clip_by_global_norm/_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_283" op: "AddV2" input: "Mul_451" input: "Mul_452" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_453/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_453/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_453" op: "Mul" input: "Mul_453/x" input: "Mul_453/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_83" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_454/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_454" op: "Mul" input: "Mul_454/x" input: "Square_83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_284" op: "AddV2" input: "Mul_453" input: "Mul_454" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_83" op: "Sqrt" input: "add_284" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_285/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_285" op: "AddV2" input: "Sqrt_83" input: "add_285/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_84" op: "RealDiv" input: "add_283" input: "add_285" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_455" op: "Mul" input: "add" input: "truediv_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_365" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_84" op: "Sub" input: "ReadVariableOp_365" input: "mul_455" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_398" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma" input: "sub_84" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_366" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma" input: "^AssignVariableOp_398" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_399" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" input: "add_283" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_367" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" input: "^AssignVariableOp_399" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_400" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" input: "add_284" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_368" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" input: "^AssignVariableOp_400" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_456/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_456/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_456" op: "Mul" input: "Mul_456/x" input: "Mul_456/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_457/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_457" op: "Mul" input: "Mul_457/x" input: "clip_by_global_norm/clip_by_global_norm/_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_286" op: "AddV2" input: "Mul_456" input: "Mul_457" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_458/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_458/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_458" op: "Mul" input: "Mul_458/x" input: "Mul_458/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_84" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_459/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_459" op: "Mul" input: "Mul_459/x" input: "Square_84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_287" op: "AddV2" input: "Mul_458" input: "Mul_459" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_84" op: "Sqrt" input: "add_287" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_288/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_288" op: "AddV2" input: "Sqrt_84" input: "add_288/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_85" op: "RealDiv" input: "add_286" input: "add_288" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_460" op: "Mul" input: "add" input: "truediv_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_369" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_85" op: "Sub" input: "ReadVariableOp_369" input: "mul_460" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_401" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta" input: "sub_85" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_370" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta" input: "^AssignVariableOp_401" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_402" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" input: "add_286" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_371" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" input: "^AssignVariableOp_402" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_403" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" input: "add_287" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_372" op: "ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" input: "^AssignVariableOp_403" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_461/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_461/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_461" op: "Mul" input: "Mul_461/x" input: "Mul_461/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_462/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_462" op: "Mul" input: "Mul_462/x" input: "clip_by_global_norm/clip_by_global_norm/_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_289" op: "AddV2" input: "Mul_461" input: "Mul_462" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_463/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_463/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_463" op: "Mul" input: "Mul_463/x" input: "Mul_463/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_85" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_464/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_464" op: "Mul" input: "Mul_464/x" input: "Square_85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_290" op: "AddV2" input: "Mul_463" input: "Mul_464" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_85" op: "Sqrt" input: "add_290" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_291/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_291" op: "AddV2" input: "Sqrt_85" input: "add_291/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_86" op: "RealDiv" input: "add_289" input: "add_291" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_373" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_465/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_465" op: "Mul" input: "mul_465/x" input: "ReadVariableOp_373" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_292" op: "AddV2" input: "truediv_86" input: "mul_465" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_466" op: "Mul" input: "add" input: "add_292" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_374" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_86" op: "Sub" input: "ReadVariableOp_374" input: "mul_466" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_404" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel" input: "sub_86" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_375" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel" input: "^AssignVariableOp_404" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_405" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" input: "add_289" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_376" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" input: "^AssignVariableOp_405" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_406" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" input: "add_290" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_377" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" input: "^AssignVariableOp_406" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/query/bias/adam_m" } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/query/bias/adam_v" } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/query/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_467/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_467/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_467" op: "Mul" input: "Mul_467/x" input: "Mul_467/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_468/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_468" op: "Mul" input: "Mul_468/x" input: "clip_by_global_norm/clip_by_global_norm/_86" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_293" op: "AddV2" input: "Mul_467" input: "Mul_468" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_469/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_469/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_469" op: "Mul" input: "Mul_469/x" input: "Mul_469/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_86" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_86" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_470/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_470" op: "Mul" input: "Mul_470/x" input: "Square_86" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_294" op: "AddV2" input: "Mul_469" input: "Mul_470" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_86" op: "Sqrt" input: "add_294" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_295/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_295" op: "AddV2" input: "Sqrt_86" input: "add_295/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_87" op: "RealDiv" input: "add_293" input: "add_295" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_471" op: "Mul" input: "add" input: "truediv_87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_378" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_87" op: "Sub" input: "ReadVariableOp_378" input: "mul_471" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_407" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias" input: "sub_87" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_379" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias" input: "^AssignVariableOp_407" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_408" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m" input: "add_293" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_380" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m" input: "^AssignVariableOp_408" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_409" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v" input: "add_294" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_381" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v" input: "^AssignVariableOp_409" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_472/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_472/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_472" op: "Mul" input: "Mul_472/x" input: "Mul_472/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_473/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_473" op: "Mul" input: "Mul_473/x" input: "clip_by_global_norm/clip_by_global_norm/_87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_296" op: "AddV2" input: "Mul_472" input: "Mul_473" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_474/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_474/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_474" op: "Mul" input: "Mul_474/x" input: "Mul_474/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_87" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_475/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_475" op: "Mul" input: "Mul_475/x" input: "Square_87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_297" op: "AddV2" input: "Mul_474" input: "Mul_475" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_87" op: "Sqrt" input: "add_297" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_298/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_298" op: "AddV2" input: "Sqrt_87" input: "add_298/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_88" op: "RealDiv" input: "add_296" input: "add_298" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_382" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_476/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_476" op: "Mul" input: "mul_476/x" input: "ReadVariableOp_382" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_299" op: "AddV2" input: "truediv_88" input: "mul_476" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_477" op: "Mul" input: "add" input: "add_299" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_383" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_88" op: "Sub" input: "ReadVariableOp_383" input: "mul_477" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_410" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel" input: "sub_88" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_384" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel" input: "^AssignVariableOp_410" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_411" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" input: "add_296" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_385" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" input: "^AssignVariableOp_411" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_412" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" input: "add_297" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_386" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" input: "^AssignVariableOp_412" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/key/bias/adam_m" } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/key/bias/adam_v" } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/key/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_478/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_478/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_478" op: "Mul" input: "Mul_478/x" input: "Mul_478/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_479/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_479" op: "Mul" input: "Mul_479/x" input: "clip_by_global_norm/clip_by_global_norm/_88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_300" op: "AddV2" input: "Mul_478" input: "Mul_479" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_480/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_480/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_480" op: "Mul" input: "Mul_480/x" input: "Mul_480/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_88" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_481/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_481" op: "Mul" input: "Mul_481/x" input: "Square_88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_301" op: "AddV2" input: "Mul_480" input: "Mul_481" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_88" op: "Sqrt" input: "add_301" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_302/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_302" op: "AddV2" input: "Sqrt_88" input: "add_302/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_89" op: "RealDiv" input: "add_300" input: "add_302" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_482" op: "Mul" input: "add" input: "truediv_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_387" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_89" op: "Sub" input: "ReadVariableOp_387" input: "mul_482" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_413" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias" input: "sub_89" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_388" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias" input: "^AssignVariableOp_413" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_414" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m" input: "add_300" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_389" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m" input: "^AssignVariableOp_414" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_415" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v" input: "add_301" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_390" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v" input: "^AssignVariableOp_415" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_483/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_483/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_483" op: "Mul" input: "Mul_483/x" input: "Mul_483/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_484/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_484" op: "Mul" input: "Mul_484/x" input: "clip_by_global_norm/clip_by_global_norm/_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_303" op: "AddV2" input: "Mul_483" input: "Mul_484" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_485/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_485/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_485" op: "Mul" input: "Mul_485/x" input: "Mul_485/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_89" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_486/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_486" op: "Mul" input: "Mul_486/x" input: "Square_89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_304" op: "AddV2" input: "Mul_485" input: "Mul_486" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_89" op: "Sqrt" input: "add_304" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_305/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_305" op: "AddV2" input: "Sqrt_89" input: "add_305/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_90" op: "RealDiv" input: "add_303" input: "add_305" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_391" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_487/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_487" op: "Mul" input: "mul_487/x" input: "ReadVariableOp_391" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_306" op: "AddV2" input: "truediv_90" input: "mul_487" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_488" op: "Mul" input: "add" input: "add_306" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_392" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_90" op: "Sub" input: "ReadVariableOp_392" input: "mul_488" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_416" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel" input: "sub_90" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_393" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel" input: "^AssignVariableOp_416" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_417" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" input: "add_303" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_394" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" input: "^AssignVariableOp_417" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_418" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" input: "add_304" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_395" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" input: "^AssignVariableOp_418" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/value/bias/adam_m" } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/self/value/bias/adam_v" } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/self/value/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_489/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_489/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_489" op: "Mul" input: "Mul_489/x" input: "Mul_489/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_490/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_490" op: "Mul" input: "Mul_490/x" input: "clip_by_global_norm/clip_by_global_norm/_90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_307" op: "AddV2" input: "Mul_489" input: "Mul_490" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_491/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_491/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_491" op: "Mul" input: "Mul_491/x" input: "Mul_491/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_90" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_492/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_492" op: "Mul" input: "Mul_492/x" input: "Square_90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_308" op: "AddV2" input: "Mul_491" input: "Mul_492" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_90" op: "Sqrt" input: "add_308" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_309/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_309" op: "AddV2" input: "Sqrt_90" input: "add_309/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_91" op: "RealDiv" input: "add_307" input: "add_309" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_493" op: "Mul" input: "add" input: "truediv_91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_396" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_91" op: "Sub" input: "ReadVariableOp_396" input: "mul_493" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_419" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias" input: "sub_91" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_397" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias" input: "^AssignVariableOp_419" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_420" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m" input: "add_307" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_398" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m" input: "^AssignVariableOp_420" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_421" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v" input: "add_308" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_399" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v" input: "^AssignVariableOp_421" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_494/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_494/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_494" op: "Mul" input: "Mul_494/x" input: "Mul_494/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_495/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_495" op: "Mul" input: "Mul_495/x" input: "clip_by_global_norm/clip_by_global_norm/_91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_310" op: "AddV2" input: "Mul_494" input: "Mul_495" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_496/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_496/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_496" op: "Mul" input: "Mul_496/x" input: "Mul_496/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_91" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_497/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_497" op: "Mul" input: "Mul_497/x" input: "Square_91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_311" op: "AddV2" input: "Mul_496" input: "Mul_497" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_91" op: "Sqrt" input: "add_311" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_312/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_312" op: "AddV2" input: "Sqrt_91" input: "add_312/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_92" op: "RealDiv" input: "add_310" input: "add_312" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_400" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_498/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_498" op: "Mul" input: "mul_498/x" input: "ReadVariableOp_400" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_313" op: "AddV2" input: "truediv_92" input: "mul_498" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_499" op: "Mul" input: "add" input: "add_313" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_401" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_92" op: "Sub" input: "ReadVariableOp_401" input: "mul_499" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_422" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel" input: "sub_92" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_402" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel" input: "^AssignVariableOp_422" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_423" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" input: "add_310" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_403" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" input: "^AssignVariableOp_423" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_424" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" input: "add_311" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_404" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" input: "^AssignVariableOp_424" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_500/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_500/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_500" op: "Mul" input: "Mul_500/x" input: "Mul_500/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_501/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_501" op: "Mul" input: "Mul_501/x" input: "clip_by_global_norm/clip_by_global_norm/_92" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_314" op: "AddV2" input: "Mul_500" input: "Mul_501" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_502/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_502/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_502" op: "Mul" input: "Mul_502/x" input: "Mul_502/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_92" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_92" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_503/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_503" op: "Mul" input: "Mul_503/x" input: "Square_92" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_315" op: "AddV2" input: "Mul_502" input: "Mul_503" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_92" op: "Sqrt" input: "add_315" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_316/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_316" op: "AddV2" input: "Sqrt_92" input: "add_316/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_93" op: "RealDiv" input: "add_314" input: "add_316" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_504" op: "Mul" input: "add" input: "truediv_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_405" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_93" op: "Sub" input: "ReadVariableOp_405" input: "mul_504" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_425" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias" input: "sub_93" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_406" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias" input: "^AssignVariableOp_425" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_426" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" input: "add_314" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_407" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" input: "^AssignVariableOp_426" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_427" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" input: "add_315" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_408" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" input: "^AssignVariableOp_427" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_505/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_505/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_505" op: "Mul" input: "Mul_505/x" input: "Mul_505/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_506/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_506" op: "Mul" input: "Mul_506/x" input: "clip_by_global_norm/clip_by_global_norm/_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_317" op: "AddV2" input: "Mul_505" input: "Mul_506" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_507/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_507/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_507" op: "Mul" input: "Mul_507/x" input: "Mul_507/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_93" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_508/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_508" op: "Mul" input: "Mul_508/x" input: "Square_93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_318" op: "AddV2" input: "Mul_507" input: "Mul_508" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_93" op: "Sqrt" input: "add_318" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_319/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_319" op: "AddV2" input: "Sqrt_93" input: "add_319/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_94" op: "RealDiv" input: "add_317" input: "add_319" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_509" op: "Mul" input: "add" input: "truediv_94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_409" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_94" op: "Sub" input: "ReadVariableOp_409" input: "mul_509" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_428" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" input: "sub_94" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_410" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" input: "^AssignVariableOp_428" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_429" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" input: "add_317" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_411" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" input: "^AssignVariableOp_429" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_430" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" input: "add_318" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_412" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" input: "^AssignVariableOp_430" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_510/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_510/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_510" op: "Mul" input: "Mul_510/x" input: "Mul_510/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_511/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_511" op: "Mul" input: "Mul_511/x" input: "clip_by_global_norm/clip_by_global_norm/_94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_320" op: "AddV2" input: "Mul_510" input: "Mul_511" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_512/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_512/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_512" op: "Mul" input: "Mul_512/x" input: "Mul_512/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_94" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_513/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_513" op: "Mul" input: "Mul_513/x" input: "Square_94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_321" op: "AddV2" input: "Mul_512" input: "Mul_513" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_94" op: "Sqrt" input: "add_321" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_322/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_322" op: "AddV2" input: "Sqrt_94" input: "add_322/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_95" op: "RealDiv" input: "add_320" input: "add_322" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_514" op: "Mul" input: "add" input: "truediv_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_413" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_95" op: "Sub" input: "ReadVariableOp_413" input: "mul_514" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_431" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta" input: "sub_95" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_414" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta" input: "^AssignVariableOp_431" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_432" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" input: "add_320" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_415" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" input: "^AssignVariableOp_432" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_433" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" input: "add_321" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_416" op: "ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" input: "^AssignVariableOp_433" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_515/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_515/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_515" op: "Mul" input: "Mul_515/x" input: "Mul_515/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_516/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_516" op: "Mul" input: "Mul_516/x" input: "clip_by_global_norm/clip_by_global_norm/_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_323" op: "AddV2" input: "Mul_515" input: "Mul_516" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_517/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_517/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_517" op: "Mul" input: "Mul_517/x" input: "Mul_517/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_95" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_518/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_518" op: "Mul" input: "Mul_518/x" input: "Square_95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_324" op: "AddV2" input: "Mul_517" input: "Mul_518" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_95" op: "Sqrt" input: "add_324" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_325/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_325" op: "AddV2" input: "Sqrt_95" input: "add_325/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_96" op: "RealDiv" input: "add_323" input: "add_325" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_417" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_519/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_519" op: "Mul" input: "mul_519/x" input: "ReadVariableOp_417" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_326" op: "AddV2" input: "truediv_96" input: "mul_519" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_520" op: "Mul" input: "add" input: "add_326" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_418" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_96" op: "Sub" input: "ReadVariableOp_418" input: "mul_520" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "AssignVariableOp_434" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel" input: "sub_96" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_419" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel" input: "^AssignVariableOp_434" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_435" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" input: "add_323" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_420" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" input: "^AssignVariableOp_435" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_436" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" input: "add_324" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_421" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" input: "^AssignVariableOp_436" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_521/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_521/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_521" op: "Mul" input: "Mul_521/x" input: "Mul_521/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_522/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_522" op: "Mul" input: "Mul_522/x" input: "clip_by_global_norm/clip_by_global_norm/_96" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_327" op: "AddV2" input: "Mul_521" input: "Mul_522" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_523/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_523/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_523" op: "Mul" input: "Mul_523/x" input: "Mul_523/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_96" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_96" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_524/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_524" op: "Mul" input: "Mul_524/x" input: "Square_96" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_328" op: "AddV2" input: "Mul_523" input: "Mul_524" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_96" op: "Sqrt" input: "add_328" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_329/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_329" op: "AddV2" input: "Sqrt_96" input: "add_329/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_97" op: "RealDiv" input: "add_327" input: "add_329" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_525" op: "Mul" input: "add" input: "truediv_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "ReadVariableOp_422" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_97" op: "Sub" input: "ReadVariableOp_422" input: "mul_525" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "AssignVariableOp_437" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias" input: "sub_97" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_423" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias" input: "^AssignVariableOp_437" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_438" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" input: "add_327" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_424" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" input: "^AssignVariableOp_438" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_439" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" input: "add_328" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_425" op: "ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" input: "^AssignVariableOp_439" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_m" input: "bert/encoder/layer_5/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_v" input: "bert/encoder/layer_5/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_526/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_526/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_526" op: "Mul" input: "Mul_526/x" input: "Mul_526/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_527/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_527" op: "Mul" input: "Mul_527/x" input: "clip_by_global_norm/clip_by_global_norm/_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_330" op: "AddV2" input: "Mul_526" input: "Mul_527" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_528/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_528/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_528" op: "Mul" input: "Mul_528/x" input: "Mul_528/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_97" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_529/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_529" op: "Mul" input: "Mul_529/x" input: "Square_97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_331" op: "AddV2" input: "Mul_528" input: "Mul_529" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_97" op: "Sqrt" input: "add_331" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_332/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_332" op: "AddV2" input: "Sqrt_97" input: "add_332/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_98" op: "RealDiv" input: "add_330" input: "add_332" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_426" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_530/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_530" op: "Mul" input: "mul_530/x" input: "ReadVariableOp_426" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_333" op: "AddV2" input: "truediv_98" input: "mul_530" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_531" op: "Mul" input: "add" input: "add_333" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_427" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_98" op: "Sub" input: "ReadVariableOp_427" input: "mul_531" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_440" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/kernel" input: "sub_98" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_428" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/kernel" input: "^AssignVariableOp_440" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_441" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_m" input: "add_330" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_429" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_m" input: "^AssignVariableOp_441" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_442" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_v" input: "add_331" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_430" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_v" input: "^AssignVariableOp_442" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/bias/adam_m" input: "bert/encoder/layer_5/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/bias/adam_v" input: "bert/encoder/layer_5/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_532/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_532/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_532" op: "Mul" input: "Mul_532/x" input: "Mul_532/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_533/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_533" op: "Mul" input: "Mul_533/x" input: "clip_by_global_norm/clip_by_global_norm/_98" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_334" op: "AddV2" input: "Mul_532" input: "Mul_533" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_534/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_534/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_534" op: "Mul" input: "Mul_534/x" input: "Mul_534/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_98" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_98" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_535/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_535" op: "Mul" input: "Mul_535/x" input: "Square_98" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_335" op: "AddV2" input: "Mul_534" input: "Mul_535" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_98" op: "Sqrt" input: "add_335" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_336/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_336" op: "AddV2" input: "Sqrt_98" input: "add_336/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_99" op: "RealDiv" input: "add_334" input: "add_336" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_536" op: "Mul" input: "add" input: "truediv_99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_431" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_99" op: "Sub" input: "ReadVariableOp_431" input: "mul_536" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_443" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/bias" input: "sub_99" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_432" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/bias" input: "^AssignVariableOp_443" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_444" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/bias/adam_m" input: "add_334" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_433" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/bias/adam_m" input: "^AssignVariableOp_444" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_445" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/bias/adam_v" input: "add_335" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_434" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/dense/bias/adam_v" input: "^AssignVariableOp_445" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_537/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_537/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_537" op: "Mul" input: "Mul_537/x" input: "Mul_537/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_538/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_538" op: "Mul" input: "Mul_538/x" input: "clip_by_global_norm/clip_by_global_norm/_99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_337" op: "AddV2" input: "Mul_537" input: "Mul_538" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_539/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_539/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_539" op: "Mul" input: "Mul_539/x" input: "Mul_539/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_99" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_540/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_540" op: "Mul" input: "Mul_540/x" input: "Square_99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_338" op: "AddV2" input: "Mul_539" input: "Mul_540" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_99" op: "Sqrt" input: "add_338" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_339/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_339" op: "AddV2" input: "Sqrt_99" input: "add_339/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_100" op: "RealDiv" input: "add_337" input: "add_339" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_541" op: "Mul" input: "add" input: "truediv_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_435" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_100" op: "Sub" input: "ReadVariableOp_435" input: "mul_541" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_446" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma" input: "sub_100" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_436" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma" input: "^AssignVariableOp_446" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_447" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" input: "add_337" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_437" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" input: "^AssignVariableOp_447" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_448" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" input: "add_338" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_438" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" input: "^AssignVariableOp_448" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_542/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_542/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_542" op: "Mul" input: "Mul_542/x" input: "Mul_542/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_543/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_543" op: "Mul" input: "Mul_543/x" input: "clip_by_global_norm/clip_by_global_norm/_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_340" op: "AddV2" input: "Mul_542" input: "Mul_543" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_544/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_544/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_544" op: "Mul" input: "Mul_544/x" input: "Mul_544/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_100" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_545/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_545" op: "Mul" input: "Mul_545/x" input: "Square_100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_341" op: "AddV2" input: "Mul_544" input: "Mul_545" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_100" op: "Sqrt" input: "add_341" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_342/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_342" op: "AddV2" input: "Sqrt_100" input: "add_342/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_101" op: "RealDiv" input: "add_340" input: "add_342" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_546" op: "Mul" input: "add" input: "truediv_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_439" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_101" op: "Sub" input: "ReadVariableOp_439" input: "mul_546" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_449" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta" input: "sub_101" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_440" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta" input: "^AssignVariableOp_449" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_450" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" input: "add_340" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_441" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" input: "^AssignVariableOp_450" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_451" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" input: "add_341" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_442" op: "ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" input: "^AssignVariableOp_451" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_547/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_547/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_547" op: "Mul" input: "Mul_547/x" input: "Mul_547/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_548/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_548" op: "Mul" input: "Mul_548/x" input: "clip_by_global_norm/clip_by_global_norm/_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_343" op: "AddV2" input: "Mul_547" input: "Mul_548" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_549/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_549/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_549" op: "Mul" input: "Mul_549/x" input: "Mul_549/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_101" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_550/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_550" op: "Mul" input: "Mul_550/x" input: "Square_101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_344" op: "AddV2" input: "Mul_549" input: "Mul_550" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_101" op: "Sqrt" input: "add_344" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_345/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_345" op: "AddV2" input: "Sqrt_101" input: "add_345/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_102" op: "RealDiv" input: "add_343" input: "add_345" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_443" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_551/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_551" op: "Mul" input: "mul_551/x" input: "ReadVariableOp_443" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_346" op: "AddV2" input: "truediv_102" input: "mul_551" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_552" op: "Mul" input: "add" input: "add_346" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_444" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_102" op: "Sub" input: "ReadVariableOp_444" input: "mul_552" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_452" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel" input: "sub_102" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_445" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel" input: "^AssignVariableOp_452" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_453" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" input: "add_343" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_446" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" input: "^AssignVariableOp_453" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_454" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" input: "add_344" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_447" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" input: "^AssignVariableOp_454" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/query/bias/adam_m" } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/query/bias/adam_v" } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/query/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_553/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_553/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_553" op: "Mul" input: "Mul_553/x" input: "Mul_553/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_554/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_554" op: "Mul" input: "Mul_554/x" input: "clip_by_global_norm/clip_by_global_norm/_102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_347" op: "AddV2" input: "Mul_553" input: "Mul_554" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_555/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_555/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_555" op: "Mul" input: "Mul_555/x" input: "Mul_555/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_102" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_556/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_556" op: "Mul" input: "Mul_556/x" input: "Square_102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_348" op: "AddV2" input: "Mul_555" input: "Mul_556" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_102" op: "Sqrt" input: "add_348" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_349/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_349" op: "AddV2" input: "Sqrt_102" input: "add_349/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_103" op: "RealDiv" input: "add_347" input: "add_349" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_557" op: "Mul" input: "add" input: "truediv_103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_448" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_103" op: "Sub" input: "ReadVariableOp_448" input: "mul_557" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_455" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias" input: "sub_103" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_449" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias" input: "^AssignVariableOp_455" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_456" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m" input: "add_347" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_450" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m" input: "^AssignVariableOp_456" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_457" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v" input: "add_348" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_451" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v" input: "^AssignVariableOp_457" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_558/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_558/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_558" op: "Mul" input: "Mul_558/x" input: "Mul_558/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_559/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_559" op: "Mul" input: "Mul_559/x" input: "clip_by_global_norm/clip_by_global_norm/_103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_350" op: "AddV2" input: "Mul_558" input: "Mul_559" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_560/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_560/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_560" op: "Mul" input: "Mul_560/x" input: "Mul_560/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_103" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_561/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_561" op: "Mul" input: "Mul_561/x" input: "Square_103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_351" op: "AddV2" input: "Mul_560" input: "Mul_561" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_103" op: "Sqrt" input: "add_351" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_352/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_352" op: "AddV2" input: "Sqrt_103" input: "add_352/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_104" op: "RealDiv" input: "add_350" input: "add_352" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_452" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_562/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_562" op: "Mul" input: "mul_562/x" input: "ReadVariableOp_452" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_353" op: "AddV2" input: "truediv_104" input: "mul_562" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_563" op: "Mul" input: "add" input: "add_353" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_453" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_104" op: "Sub" input: "ReadVariableOp_453" input: "mul_563" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_458" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel" input: "sub_104" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_454" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel" input: "^AssignVariableOp_458" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_459" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" input: "add_350" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_455" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" input: "^AssignVariableOp_459" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_460" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" input: "add_351" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_456" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" input: "^AssignVariableOp_460" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/key/bias/adam_m" } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/key/bias/adam_v" } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/key/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_564/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_564/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_564" op: "Mul" input: "Mul_564/x" input: "Mul_564/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_565/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_565" op: "Mul" input: "Mul_565/x" input: "clip_by_global_norm/clip_by_global_norm/_104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_354" op: "AddV2" input: "Mul_564" input: "Mul_565" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_566/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_566/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_566" op: "Mul" input: "Mul_566/x" input: "Mul_566/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_104" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_567/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_567" op: "Mul" input: "Mul_567/x" input: "Square_104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_355" op: "AddV2" input: "Mul_566" input: "Mul_567" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_104" op: "Sqrt" input: "add_355" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_356/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_356" op: "AddV2" input: "Sqrt_104" input: "add_356/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_105" op: "RealDiv" input: "add_354" input: "add_356" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_568" op: "Mul" input: "add" input: "truediv_105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_457" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_105" op: "Sub" input: "ReadVariableOp_457" input: "mul_568" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_461" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias" input: "sub_105" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_458" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias" input: "^AssignVariableOp_461" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_462" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m" input: "add_354" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_459" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m" input: "^AssignVariableOp_462" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_463" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v" input: "add_355" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_460" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v" input: "^AssignVariableOp_463" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_569/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_569/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_569" op: "Mul" input: "Mul_569/x" input: "Mul_569/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_570/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_570" op: "Mul" input: "Mul_570/x" input: "clip_by_global_norm/clip_by_global_norm/_105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_357" op: "AddV2" input: "Mul_569" input: "Mul_570" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_571/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_571/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_571" op: "Mul" input: "Mul_571/x" input: "Mul_571/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_105" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_572/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_572" op: "Mul" input: "Mul_572/x" input: "Square_105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_358" op: "AddV2" input: "Mul_571" input: "Mul_572" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_105" op: "Sqrt" input: "add_358" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_359/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_359" op: "AddV2" input: "Sqrt_105" input: "add_359/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_106" op: "RealDiv" input: "add_357" input: "add_359" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_461" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_573/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_573" op: "Mul" input: "mul_573/x" input: "ReadVariableOp_461" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_360" op: "AddV2" input: "truediv_106" input: "mul_573" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_574" op: "Mul" input: "add" input: "add_360" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_462" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_106" op: "Sub" input: "ReadVariableOp_462" input: "mul_574" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_464" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel" input: "sub_106" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_463" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel" input: "^AssignVariableOp_464" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_465" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" input: "add_357" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_464" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" input: "^AssignVariableOp_465" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_466" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" input: "add_358" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_465" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" input: "^AssignVariableOp_466" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/value/bias/adam_m" } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/self/value/bias/adam_v" } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/self/value/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_575/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_575/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_575" op: "Mul" input: "Mul_575/x" input: "Mul_575/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_576/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_576" op: "Mul" input: "Mul_576/x" input: "clip_by_global_norm/clip_by_global_norm/_106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_361" op: "AddV2" input: "Mul_575" input: "Mul_576" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_577/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_577/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_577" op: "Mul" input: "Mul_577/x" input: "Mul_577/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_106" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_578/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_578" op: "Mul" input: "Mul_578/x" input: "Square_106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_362" op: "AddV2" input: "Mul_577" input: "Mul_578" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_106" op: "Sqrt" input: "add_362" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_363/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_363" op: "AddV2" input: "Sqrt_106" input: "add_363/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_107" op: "RealDiv" input: "add_361" input: "add_363" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_579" op: "Mul" input: "add" input: "truediv_107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_466" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_107" op: "Sub" input: "ReadVariableOp_466" input: "mul_579" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_467" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias" input: "sub_107" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_467" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias" input: "^AssignVariableOp_467" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_468" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m" input: "add_361" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_468" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m" input: "^AssignVariableOp_468" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_469" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v" input: "add_362" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_469" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v" input: "^AssignVariableOp_469" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_580/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_580/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_580" op: "Mul" input: "Mul_580/x" input: "Mul_580/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_581/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_581" op: "Mul" input: "Mul_581/x" input: "clip_by_global_norm/clip_by_global_norm/_107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_364" op: "AddV2" input: "Mul_580" input: "Mul_581" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_582/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_582/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_582" op: "Mul" input: "Mul_582/x" input: "Mul_582/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_107" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_583/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_583" op: "Mul" input: "Mul_583/x" input: "Square_107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_365" op: "AddV2" input: "Mul_582" input: "Mul_583" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_107" op: "Sqrt" input: "add_365" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_366/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_366" op: "AddV2" input: "Sqrt_107" input: "add_366/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_108" op: "RealDiv" input: "add_364" input: "add_366" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_470" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_584/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_584" op: "Mul" input: "mul_584/x" input: "ReadVariableOp_470" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_367" op: "AddV2" input: "truediv_108" input: "mul_584" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_585" op: "Mul" input: "add" input: "add_367" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_471" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_108" op: "Sub" input: "ReadVariableOp_471" input: "mul_585" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_470" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel" input: "sub_108" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_472" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel" input: "^AssignVariableOp_470" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_471" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" input: "add_364" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_473" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" input: "^AssignVariableOp_471" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_472" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" input: "add_365" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_474" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" input: "^AssignVariableOp_472" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_586/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_586/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_586" op: "Mul" input: "Mul_586/x" input: "Mul_586/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_587/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_587" op: "Mul" input: "Mul_587/x" input: "clip_by_global_norm/clip_by_global_norm/_108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_368" op: "AddV2" input: "Mul_586" input: "Mul_587" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_588/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_588/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_588" op: "Mul" input: "Mul_588/x" input: "Mul_588/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_108" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_589/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_589" op: "Mul" input: "Mul_589/x" input: "Square_108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_369" op: "AddV2" input: "Mul_588" input: "Mul_589" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_108" op: "Sqrt" input: "add_369" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_370/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_370" op: "AddV2" input: "Sqrt_108" input: "add_370/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_109" op: "RealDiv" input: "add_368" input: "add_370" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_590" op: "Mul" input: "add" input: "truediv_109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_475" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_109" op: "Sub" input: "ReadVariableOp_475" input: "mul_590" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_473" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias" input: "sub_109" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_476" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias" input: "^AssignVariableOp_473" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_474" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" input: "add_368" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_477" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" input: "^AssignVariableOp_474" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_475" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" input: "add_369" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_478" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" input: "^AssignVariableOp_475" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_591/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_591/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_591" op: "Mul" input: "Mul_591/x" input: "Mul_591/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_592/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_592" op: "Mul" input: "Mul_592/x" input: "clip_by_global_norm/clip_by_global_norm/_109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_371" op: "AddV2" input: "Mul_591" input: "Mul_592" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_593/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_593/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_593" op: "Mul" input: "Mul_593/x" input: "Mul_593/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_109" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_594/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_594" op: "Mul" input: "Mul_594/x" input: "Square_109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_372" op: "AddV2" input: "Mul_593" input: "Mul_594" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_109" op: "Sqrt" input: "add_372" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_373/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_373" op: "AddV2" input: "Sqrt_109" input: "add_373/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_110" op: "RealDiv" input: "add_371" input: "add_373" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_595" op: "Mul" input: "add" input: "truediv_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_479" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_110" op: "Sub" input: "ReadVariableOp_479" input: "mul_595" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_476" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" input: "sub_110" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_480" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" input: "^AssignVariableOp_476" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_477" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" input: "add_371" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_481" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" input: "^AssignVariableOp_477" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_478" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" input: "add_372" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_482" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" input: "^AssignVariableOp_478" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_596/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_596/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_596" op: "Mul" input: "Mul_596/x" input: "Mul_596/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_597/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_597" op: "Mul" input: "Mul_597/x" input: "clip_by_global_norm/clip_by_global_norm/_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_374" op: "AddV2" input: "Mul_596" input: "Mul_597" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_598/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_598/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_598" op: "Mul" input: "Mul_598/x" input: "Mul_598/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_110" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_599/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_599" op: "Mul" input: "Mul_599/x" input: "Square_110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_375" op: "AddV2" input: "Mul_598" input: "Mul_599" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_110" op: "Sqrt" input: "add_375" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_376/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_376" op: "AddV2" input: "Sqrt_110" input: "add_376/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_111" op: "RealDiv" input: "add_374" input: "add_376" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_600" op: "Mul" input: "add" input: "truediv_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_483" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_111" op: "Sub" input: "ReadVariableOp_483" input: "mul_600" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_479" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta" input: "sub_111" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_484" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta" input: "^AssignVariableOp_479" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_480" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" input: "add_374" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_485" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" input: "^AssignVariableOp_480" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_481" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" input: "add_375" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_486" op: "ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" input: "^AssignVariableOp_481" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_601/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_601/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_601" op: "Mul" input: "Mul_601/x" input: "Mul_601/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_602/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_602" op: "Mul" input: "Mul_602/x" input: "clip_by_global_norm/clip_by_global_norm/_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_377" op: "AddV2" input: "Mul_601" input: "Mul_602" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_603/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_603/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_603" op: "Mul" input: "Mul_603/x" input: "Mul_603/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_111" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_604/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_604" op: "Mul" input: "Mul_604/x" input: "Square_111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_378" op: "AddV2" input: "Mul_603" input: "Mul_604" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_111" op: "Sqrt" input: "add_378" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_379/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_379" op: "AddV2" input: "Sqrt_111" input: "add_379/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_112" op: "RealDiv" input: "add_377" input: "add_379" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_487" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_605/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_605" op: "Mul" input: "mul_605/x" input: "ReadVariableOp_487" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_380" op: "AddV2" input: "truediv_112" input: "mul_605" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_606" op: "Mul" input: "add" input: "add_380" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_488" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_112" op: "Sub" input: "ReadVariableOp_488" input: "mul_606" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "AssignVariableOp_482" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel" input: "sub_112" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_489" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel" input: "^AssignVariableOp_482" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_483" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" input: "add_377" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_490" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" input: "^AssignVariableOp_483" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_484" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" input: "add_378" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_491" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" input: "^AssignVariableOp_484" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_607/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_607/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_607" op: "Mul" input: "Mul_607/x" input: "Mul_607/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_608/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_608" op: "Mul" input: "Mul_608/x" input: "clip_by_global_norm/clip_by_global_norm/_112" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_381" op: "AddV2" input: "Mul_607" input: "Mul_608" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_609/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_609/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_609" op: "Mul" input: "Mul_609/x" input: "Mul_609/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_112" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_112" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_610/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_610" op: "Mul" input: "Mul_610/x" input: "Square_112" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_382" op: "AddV2" input: "Mul_609" input: "Mul_610" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_112" op: "Sqrt" input: "add_382" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_383/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_383" op: "AddV2" input: "Sqrt_112" input: "add_383/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_113" op: "RealDiv" input: "add_381" input: "add_383" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_611" op: "Mul" input: "add" input: "truediv_113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "ReadVariableOp_492" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_113" op: "Sub" input: "ReadVariableOp_492" input: "mul_611" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "AssignVariableOp_485" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias" input: "sub_113" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_493" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias" input: "^AssignVariableOp_485" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_486" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" input: "add_381" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_494" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" input: "^AssignVariableOp_486" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_487" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" input: "add_382" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_495" op: "ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" input: "^AssignVariableOp_487" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_m" input: "bert/encoder/layer_6/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_v" input: "bert/encoder/layer_6/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_612/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_612/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_612" op: "Mul" input: "Mul_612/x" input: "Mul_612/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_613/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_613" op: "Mul" input: "Mul_613/x" input: "clip_by_global_norm/clip_by_global_norm/_113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_384" op: "AddV2" input: "Mul_612" input: "Mul_613" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_614/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_614/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_614" op: "Mul" input: "Mul_614/x" input: "Mul_614/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_113" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_615/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_615" op: "Mul" input: "Mul_615/x" input: "Square_113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_385" op: "AddV2" input: "Mul_614" input: "Mul_615" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_113" op: "Sqrt" input: "add_385" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_386/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_386" op: "AddV2" input: "Sqrt_113" input: "add_386/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_114" op: "RealDiv" input: "add_384" input: "add_386" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_496" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_616/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_616" op: "Mul" input: "mul_616/x" input: "ReadVariableOp_496" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_387" op: "AddV2" input: "truediv_114" input: "mul_616" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_617" op: "Mul" input: "add" input: "add_387" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_497" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_114" op: "Sub" input: "ReadVariableOp_497" input: "mul_617" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_488" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/kernel" input: "sub_114" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_498" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/kernel" input: "^AssignVariableOp_488" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_489" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_m" input: "add_384" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_499" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_m" input: "^AssignVariableOp_489" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_490" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_v" input: "add_385" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_500" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_v" input: "^AssignVariableOp_490" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/bias/adam_m" input: "bert/encoder/layer_6/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/bias/adam_v" input: "bert/encoder/layer_6/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_618/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_618/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_618" op: "Mul" input: "Mul_618/x" input: "Mul_618/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_619/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_619" op: "Mul" input: "Mul_619/x" input: "clip_by_global_norm/clip_by_global_norm/_114" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_388" op: "AddV2" input: "Mul_618" input: "Mul_619" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_620/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_620/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_620" op: "Mul" input: "Mul_620/x" input: "Mul_620/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_114" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_114" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_621/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_621" op: "Mul" input: "Mul_621/x" input: "Square_114" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_389" op: "AddV2" input: "Mul_620" input: "Mul_621" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_114" op: "Sqrt" input: "add_389" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_390/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_390" op: "AddV2" input: "Sqrt_114" input: "add_390/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_115" op: "RealDiv" input: "add_388" input: "add_390" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_622" op: "Mul" input: "add" input: "truediv_115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_501" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_115" op: "Sub" input: "ReadVariableOp_501" input: "mul_622" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_491" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/bias" input: "sub_115" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_502" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/bias" input: "^AssignVariableOp_491" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_492" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/bias/adam_m" input: "add_388" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_503" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/bias/adam_m" input: "^AssignVariableOp_492" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_493" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/bias/adam_v" input: "add_389" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_504" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/dense/bias/adam_v" input: "^AssignVariableOp_493" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_623/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_623/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_623" op: "Mul" input: "Mul_623/x" input: "Mul_623/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_624/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_624" op: "Mul" input: "Mul_624/x" input: "clip_by_global_norm/clip_by_global_norm/_115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_391" op: "AddV2" input: "Mul_623" input: "Mul_624" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_625/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_625/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_625" op: "Mul" input: "Mul_625/x" input: "Mul_625/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_115" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_626/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_626" op: "Mul" input: "Mul_626/x" input: "Square_115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_392" op: "AddV2" input: "Mul_625" input: "Mul_626" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_115" op: "Sqrt" input: "add_392" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_393/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_393" op: "AddV2" input: "Sqrt_115" input: "add_393/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_116" op: "RealDiv" input: "add_391" input: "add_393" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_627" op: "Mul" input: "add" input: "truediv_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_505" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_116" op: "Sub" input: "ReadVariableOp_505" input: "mul_627" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_494" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma" input: "sub_116" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_506" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma" input: "^AssignVariableOp_494" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_495" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" input: "add_391" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_507" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" input: "^AssignVariableOp_495" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_496" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" input: "add_392" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_508" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" input: "^AssignVariableOp_496" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_628/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_628/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_628" op: "Mul" input: "Mul_628/x" input: "Mul_628/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_629/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_629" op: "Mul" input: "Mul_629/x" input: "clip_by_global_norm/clip_by_global_norm/_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_394" op: "AddV2" input: "Mul_628" input: "Mul_629" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_630/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_630/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_630" op: "Mul" input: "Mul_630/x" input: "Mul_630/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_116" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_631/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_631" op: "Mul" input: "Mul_631/x" input: "Square_116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_395" op: "AddV2" input: "Mul_630" input: "Mul_631" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_116" op: "Sqrt" input: "add_395" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_396/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_396" op: "AddV2" input: "Sqrt_116" input: "add_396/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_117" op: "RealDiv" input: "add_394" input: "add_396" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_632" op: "Mul" input: "add" input: "truediv_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_509" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_117" op: "Sub" input: "ReadVariableOp_509" input: "mul_632" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_497" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta" input: "sub_117" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_510" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta" input: "^AssignVariableOp_497" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_498" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" input: "add_394" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_511" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" input: "^AssignVariableOp_498" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_499" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" input: "add_395" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_512" op: "ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" input: "^AssignVariableOp_499" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_633/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_633/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_633" op: "Mul" input: "Mul_633/x" input: "Mul_633/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_634/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_634" op: "Mul" input: "Mul_634/x" input: "clip_by_global_norm/clip_by_global_norm/_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_397" op: "AddV2" input: "Mul_633" input: "Mul_634" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_635/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_635/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_635" op: "Mul" input: "Mul_635/x" input: "Mul_635/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_117" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_636/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_636" op: "Mul" input: "Mul_636/x" input: "Square_117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_398" op: "AddV2" input: "Mul_635" input: "Mul_636" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_117" op: "Sqrt" input: "add_398" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_399/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_399" op: "AddV2" input: "Sqrt_117" input: "add_399/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_118" op: "RealDiv" input: "add_397" input: "add_399" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_513" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_637/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_637" op: "Mul" input: "mul_637/x" input: "ReadVariableOp_513" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_400" op: "AddV2" input: "truediv_118" input: "mul_637" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_638" op: "Mul" input: "add" input: "add_400" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_514" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_118" op: "Sub" input: "ReadVariableOp_514" input: "mul_638" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_500" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel" input: "sub_118" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_515" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel" input: "^AssignVariableOp_500" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_501" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" input: "add_397" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_516" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" input: "^AssignVariableOp_501" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_502" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" input: "add_398" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_517" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" input: "^AssignVariableOp_502" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/query/bias/adam_m" } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/query/bias/adam_v" } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/query/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_639/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_639/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_639" op: "Mul" input: "Mul_639/x" input: "Mul_639/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_640/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_640" op: "Mul" input: "Mul_640/x" input: "clip_by_global_norm/clip_by_global_norm/_118" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_401" op: "AddV2" input: "Mul_639" input: "Mul_640" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_641/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_641/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_641" op: "Mul" input: "Mul_641/x" input: "Mul_641/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_118" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_118" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_642/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_642" op: "Mul" input: "Mul_642/x" input: "Square_118" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_402" op: "AddV2" input: "Mul_641" input: "Mul_642" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_118" op: "Sqrt" input: "add_402" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_403/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_403" op: "AddV2" input: "Sqrt_118" input: "add_403/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_119" op: "RealDiv" input: "add_401" input: "add_403" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_643" op: "Mul" input: "add" input: "truediv_119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_518" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_119" op: "Sub" input: "ReadVariableOp_518" input: "mul_643" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_503" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias" input: "sub_119" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_519" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias" input: "^AssignVariableOp_503" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_504" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m" input: "add_401" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_520" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m" input: "^AssignVariableOp_504" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_505" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v" input: "add_402" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_521" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v" input: "^AssignVariableOp_505" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_644/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_644/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_644" op: "Mul" input: "Mul_644/x" input: "Mul_644/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_645/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_645" op: "Mul" input: "Mul_645/x" input: "clip_by_global_norm/clip_by_global_norm/_119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_404" op: "AddV2" input: "Mul_644" input: "Mul_645" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_646/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_646/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_646" op: "Mul" input: "Mul_646/x" input: "Mul_646/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_119" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_647/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_647" op: "Mul" input: "Mul_647/x" input: "Square_119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_405" op: "AddV2" input: "Mul_646" input: "Mul_647" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_119" op: "Sqrt" input: "add_405" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_406/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_406" op: "AddV2" input: "Sqrt_119" input: "add_406/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_120" op: "RealDiv" input: "add_404" input: "add_406" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_522" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_648/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_648" op: "Mul" input: "mul_648/x" input: "ReadVariableOp_522" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_407" op: "AddV2" input: "truediv_120" input: "mul_648" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_649" op: "Mul" input: "add" input: "add_407" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_523" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_120" op: "Sub" input: "ReadVariableOp_523" input: "mul_649" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_506" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel" input: "sub_120" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_524" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel" input: "^AssignVariableOp_506" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_507" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" input: "add_404" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_525" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" input: "^AssignVariableOp_507" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_508" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" input: "add_405" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_526" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" input: "^AssignVariableOp_508" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/key/bias/adam_m" } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/key/bias/adam_v" } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/key/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_650/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_650/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_650" op: "Mul" input: "Mul_650/x" input: "Mul_650/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_651/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_651" op: "Mul" input: "Mul_651/x" input: "clip_by_global_norm/clip_by_global_norm/_120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_408" op: "AddV2" input: "Mul_650" input: "Mul_651" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_652/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_652/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_652" op: "Mul" input: "Mul_652/x" input: "Mul_652/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_120" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_653/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_653" op: "Mul" input: "Mul_653/x" input: "Square_120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_409" op: "AddV2" input: "Mul_652" input: "Mul_653" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_120" op: "Sqrt" input: "add_409" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_410/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_410" op: "AddV2" input: "Sqrt_120" input: "add_410/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_121" op: "RealDiv" input: "add_408" input: "add_410" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_654" op: "Mul" input: "add" input: "truediv_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_527" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_121" op: "Sub" input: "ReadVariableOp_527" input: "mul_654" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_509" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias" input: "sub_121" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_528" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias" input: "^AssignVariableOp_509" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_510" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m" input: "add_408" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_529" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m" input: "^AssignVariableOp_510" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_511" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v" input: "add_409" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_530" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v" input: "^AssignVariableOp_511" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_655/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_655/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_655" op: "Mul" input: "Mul_655/x" input: "Mul_655/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_656/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_656" op: "Mul" input: "Mul_656/x" input: "clip_by_global_norm/clip_by_global_norm/_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_411" op: "AddV2" input: "Mul_655" input: "Mul_656" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_657/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_657/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_657" op: "Mul" input: "Mul_657/x" input: "Mul_657/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_121" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_658/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_658" op: "Mul" input: "Mul_658/x" input: "Square_121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_412" op: "AddV2" input: "Mul_657" input: "Mul_658" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_121" op: "Sqrt" input: "add_412" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_413/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_413" op: "AddV2" input: "Sqrt_121" input: "add_413/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_122" op: "RealDiv" input: "add_411" input: "add_413" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_531" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_659/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_659" op: "Mul" input: "mul_659/x" input: "ReadVariableOp_531" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_414" op: "AddV2" input: "truediv_122" input: "mul_659" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_660" op: "Mul" input: "add" input: "add_414" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_532" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_122" op: "Sub" input: "ReadVariableOp_532" input: "mul_660" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_512" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel" input: "sub_122" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_533" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel" input: "^AssignVariableOp_512" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_513" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" input: "add_411" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_534" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" input: "^AssignVariableOp_513" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_514" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" input: "add_412" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_535" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" input: "^AssignVariableOp_514" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/value/bias/adam_m" } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/self/value/bias/adam_v" } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/self/value/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_661/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_661/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_661" op: "Mul" input: "Mul_661/x" input: "Mul_661/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_662/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_662" op: "Mul" input: "Mul_662/x" input: "clip_by_global_norm/clip_by_global_norm/_122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_415" op: "AddV2" input: "Mul_661" input: "Mul_662" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_663/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_663/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_663" op: "Mul" input: "Mul_663/x" input: "Mul_663/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_122" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_664/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_664" op: "Mul" input: "Mul_664/x" input: "Square_122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_416" op: "AddV2" input: "Mul_663" input: "Mul_664" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_122" op: "Sqrt" input: "add_416" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_417/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_417" op: "AddV2" input: "Sqrt_122" input: "add_417/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_123" op: "RealDiv" input: "add_415" input: "add_417" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_665" op: "Mul" input: "add" input: "truediv_123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_536" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_123" op: "Sub" input: "ReadVariableOp_536" input: "mul_665" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_515" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias" input: "sub_123" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_537" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias" input: "^AssignVariableOp_515" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_516" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m" input: "add_415" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_538" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m" input: "^AssignVariableOp_516" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_517" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v" input: "add_416" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_539" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v" input: "^AssignVariableOp_517" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_666/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_666/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_666" op: "Mul" input: "Mul_666/x" input: "Mul_666/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_667/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_667" op: "Mul" input: "Mul_667/x" input: "clip_by_global_norm/clip_by_global_norm/_123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_418" op: "AddV2" input: "Mul_666" input: "Mul_667" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_668/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_668/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_668" op: "Mul" input: "Mul_668/x" input: "Mul_668/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_123" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_669/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_669" op: "Mul" input: "Mul_669/x" input: "Square_123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_419" op: "AddV2" input: "Mul_668" input: "Mul_669" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_123" op: "Sqrt" input: "add_419" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_420/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_420" op: "AddV2" input: "Sqrt_123" input: "add_420/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_124" op: "RealDiv" input: "add_418" input: "add_420" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_540" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_670/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_670" op: "Mul" input: "mul_670/x" input: "ReadVariableOp_540" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_421" op: "AddV2" input: "truediv_124" input: "mul_670" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_671" op: "Mul" input: "add" input: "add_421" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_541" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_124" op: "Sub" input: "ReadVariableOp_541" input: "mul_671" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_518" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel" input: "sub_124" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_542" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel" input: "^AssignVariableOp_518" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_519" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" input: "add_418" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_543" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" input: "^AssignVariableOp_519" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_520" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" input: "add_419" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_544" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" input: "^AssignVariableOp_520" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_672/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_672/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_672" op: "Mul" input: "Mul_672/x" input: "Mul_672/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_673/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_673" op: "Mul" input: "Mul_673/x" input: "clip_by_global_norm/clip_by_global_norm/_124" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_422" op: "AddV2" input: "Mul_672" input: "Mul_673" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_674/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_674/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_674" op: "Mul" input: "Mul_674/x" input: "Mul_674/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_124" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_124" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_675/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_675" op: "Mul" input: "Mul_675/x" input: "Square_124" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_423" op: "AddV2" input: "Mul_674" input: "Mul_675" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_124" op: "Sqrt" input: "add_423" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_424/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_424" op: "AddV2" input: "Sqrt_124" input: "add_424/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_125" op: "RealDiv" input: "add_422" input: "add_424" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_676" op: "Mul" input: "add" input: "truediv_125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_545" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_125" op: "Sub" input: "ReadVariableOp_545" input: "mul_676" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_521" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias" input: "sub_125" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_546" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias" input: "^AssignVariableOp_521" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_522" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" input: "add_422" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_547" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" input: "^AssignVariableOp_522" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_523" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" input: "add_423" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_548" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" input: "^AssignVariableOp_523" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_677/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_677/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_677" op: "Mul" input: "Mul_677/x" input: "Mul_677/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_678/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_678" op: "Mul" input: "Mul_678/x" input: "clip_by_global_norm/clip_by_global_norm/_125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_425" op: "AddV2" input: "Mul_677" input: "Mul_678" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_679/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_679/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_679" op: "Mul" input: "Mul_679/x" input: "Mul_679/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_125" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_680/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_680" op: "Mul" input: "Mul_680/x" input: "Square_125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_426" op: "AddV2" input: "Mul_679" input: "Mul_680" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_125" op: "Sqrt" input: "add_426" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_427/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_427" op: "AddV2" input: "Sqrt_125" input: "add_427/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_126" op: "RealDiv" input: "add_425" input: "add_427" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_681" op: "Mul" input: "add" input: "truediv_126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_549" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_126" op: "Sub" input: "ReadVariableOp_549" input: "mul_681" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_524" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" input: "sub_126" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_550" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" input: "^AssignVariableOp_524" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_525" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" input: "add_425" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_551" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" input: "^AssignVariableOp_525" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_526" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" input: "add_426" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_552" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" input: "^AssignVariableOp_526" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_682/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_682/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_682" op: "Mul" input: "Mul_682/x" input: "Mul_682/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_683/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_683" op: "Mul" input: "Mul_683/x" input: "clip_by_global_norm/clip_by_global_norm/_126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_428" op: "AddV2" input: "Mul_682" input: "Mul_683" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_684/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_684/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_684" op: "Mul" input: "Mul_684/x" input: "Mul_684/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_126" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_685/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_685" op: "Mul" input: "Mul_685/x" input: "Square_126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_429" op: "AddV2" input: "Mul_684" input: "Mul_685" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_126" op: "Sqrt" input: "add_429" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_430/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_430" op: "AddV2" input: "Sqrt_126" input: "add_430/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_127" op: "RealDiv" input: "add_428" input: "add_430" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_686" op: "Mul" input: "add" input: "truediv_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_553" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_127" op: "Sub" input: "ReadVariableOp_553" input: "mul_686" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_527" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta" input: "sub_127" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_554" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta" input: "^AssignVariableOp_527" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_528" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" input: "add_428" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_555" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" input: "^AssignVariableOp_528" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_529" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" input: "add_429" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_556" op: "ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" input: "^AssignVariableOp_529" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_687/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_687/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_687" op: "Mul" input: "Mul_687/x" input: "Mul_687/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_688/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_688" op: "Mul" input: "Mul_688/x" input: "clip_by_global_norm/clip_by_global_norm/_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_431" op: "AddV2" input: "Mul_687" input: "Mul_688" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_689/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_689/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_689" op: "Mul" input: "Mul_689/x" input: "Mul_689/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_127" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_690/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_690" op: "Mul" input: "Mul_690/x" input: "Square_127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_432" op: "AddV2" input: "Mul_689" input: "Mul_690" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_127" op: "Sqrt" input: "add_432" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_433/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_433" op: "AddV2" input: "Sqrt_127" input: "add_433/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_128" op: "RealDiv" input: "add_431" input: "add_433" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_557" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_691/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_691" op: "Mul" input: "mul_691/x" input: "ReadVariableOp_557" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_434" op: "AddV2" input: "truediv_128" input: "mul_691" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_692" op: "Mul" input: "add" input: "add_434" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_558" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_128" op: "Sub" input: "ReadVariableOp_558" input: "mul_692" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "AssignVariableOp_530" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel" input: "sub_128" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_559" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel" input: "^AssignVariableOp_530" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_531" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" input: "add_431" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_560" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" input: "^AssignVariableOp_531" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_532" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" input: "add_432" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_561" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" input: "^AssignVariableOp_532" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_693/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_693/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_693" op: "Mul" input: "Mul_693/x" input: "Mul_693/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_694/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_694" op: "Mul" input: "Mul_694/x" input: "clip_by_global_norm/clip_by_global_norm/_128" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_435" op: "AddV2" input: "Mul_693" input: "Mul_694" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_695/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_695/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_695" op: "Mul" input: "Mul_695/x" input: "Mul_695/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_128" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_128" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_696/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_696" op: "Mul" input: "Mul_696/x" input: "Square_128" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_436" op: "AddV2" input: "Mul_695" input: "Mul_696" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_128" op: "Sqrt" input: "add_436" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_437/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_437" op: "AddV2" input: "Sqrt_128" input: "add_437/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_129" op: "RealDiv" input: "add_435" input: "add_437" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_697" op: "Mul" input: "add" input: "truediv_129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "ReadVariableOp_562" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_129" op: "Sub" input: "ReadVariableOp_562" input: "mul_697" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "AssignVariableOp_533" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias" input: "sub_129" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_563" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias" input: "^AssignVariableOp_533" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_534" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" input: "add_435" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_564" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" input: "^AssignVariableOp_534" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_535" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" input: "add_436" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_565" op: "ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" input: "^AssignVariableOp_535" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_m" input: "bert/encoder/layer_7/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_v" input: "bert/encoder/layer_7/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_698/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_698/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_698" op: "Mul" input: "Mul_698/x" input: "Mul_698/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_699/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_699" op: "Mul" input: "Mul_699/x" input: "clip_by_global_norm/clip_by_global_norm/_129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_438" op: "AddV2" input: "Mul_698" input: "Mul_699" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_700/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_700/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_700" op: "Mul" input: "Mul_700/x" input: "Mul_700/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_129" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_701/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_701" op: "Mul" input: "Mul_701/x" input: "Square_129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_439" op: "AddV2" input: "Mul_700" input: "Mul_701" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_129" op: "Sqrt" input: "add_439" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_440/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_440" op: "AddV2" input: "Sqrt_129" input: "add_440/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_130" op: "RealDiv" input: "add_438" input: "add_440" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_566" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_702/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_702" op: "Mul" input: "mul_702/x" input: "ReadVariableOp_566" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_441" op: "AddV2" input: "truediv_130" input: "mul_702" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_703" op: "Mul" input: "add" input: "add_441" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_567" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_130" op: "Sub" input: "ReadVariableOp_567" input: "mul_703" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_536" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/kernel" input: "sub_130" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_568" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/kernel" input: "^AssignVariableOp_536" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_537" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_m" input: "add_438" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_569" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_m" input: "^AssignVariableOp_537" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_538" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_v" input: "add_439" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_570" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_v" input: "^AssignVariableOp_538" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/bias/adam_m" input: "bert/encoder/layer_7/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/bias/adam_v" input: "bert/encoder/layer_7/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_704/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_704/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_704" op: "Mul" input: "Mul_704/x" input: "Mul_704/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_705/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_705" op: "Mul" input: "Mul_705/x" input: "clip_by_global_norm/clip_by_global_norm/_130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_442" op: "AddV2" input: "Mul_704" input: "Mul_705" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_706/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_706/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_706" op: "Mul" input: "Mul_706/x" input: "Mul_706/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_130" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_707/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_707" op: "Mul" input: "Mul_707/x" input: "Square_130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_443" op: "AddV2" input: "Mul_706" input: "Mul_707" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_130" op: "Sqrt" input: "add_443" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_444/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_444" op: "AddV2" input: "Sqrt_130" input: "add_444/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_131" op: "RealDiv" input: "add_442" input: "add_444" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_708" op: "Mul" input: "add" input: "truediv_131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_571" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_131" op: "Sub" input: "ReadVariableOp_571" input: "mul_708" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_539" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/bias" input: "sub_131" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_572" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/bias" input: "^AssignVariableOp_539" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_540" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/bias/adam_m" input: "add_442" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_573" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/bias/adam_m" input: "^AssignVariableOp_540" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_541" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/bias/adam_v" input: "add_443" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_574" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/dense/bias/adam_v" input: "^AssignVariableOp_541" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_709/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_709/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_709" op: "Mul" input: "Mul_709/x" input: "Mul_709/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_710/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_710" op: "Mul" input: "Mul_710/x" input: "clip_by_global_norm/clip_by_global_norm/_131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_445" op: "AddV2" input: "Mul_709" input: "Mul_710" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_711/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_711/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_711" op: "Mul" input: "Mul_711/x" input: "Mul_711/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_131" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_712/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_712" op: "Mul" input: "Mul_712/x" input: "Square_131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_446" op: "AddV2" input: "Mul_711" input: "Mul_712" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_131" op: "Sqrt" input: "add_446" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_447/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_447" op: "AddV2" input: "Sqrt_131" input: "add_447/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_132" op: "RealDiv" input: "add_445" input: "add_447" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_713" op: "Mul" input: "add" input: "truediv_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_575" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_132" op: "Sub" input: "ReadVariableOp_575" input: "mul_713" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_542" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma" input: "sub_132" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_576" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma" input: "^AssignVariableOp_542" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_543" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" input: "add_445" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_577" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" input: "^AssignVariableOp_543" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_544" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" input: "add_446" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_578" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" input: "^AssignVariableOp_544" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_714/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_714/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_714" op: "Mul" input: "Mul_714/x" input: "Mul_714/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_715/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_715" op: "Mul" input: "Mul_715/x" input: "clip_by_global_norm/clip_by_global_norm/_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_448" op: "AddV2" input: "Mul_714" input: "Mul_715" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_716/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_716/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_716" op: "Mul" input: "Mul_716/x" input: "Mul_716/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_132" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_717/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_717" op: "Mul" input: "Mul_717/x" input: "Square_132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_449" op: "AddV2" input: "Mul_716" input: "Mul_717" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_132" op: "Sqrt" input: "add_449" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_450/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_450" op: "AddV2" input: "Sqrt_132" input: "add_450/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_133" op: "RealDiv" input: "add_448" input: "add_450" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_718" op: "Mul" input: "add" input: "truediv_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_579" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_133" op: "Sub" input: "ReadVariableOp_579" input: "mul_718" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_545" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta" input: "sub_133" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_580" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta" input: "^AssignVariableOp_545" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_546" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" input: "add_448" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_581" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" input: "^AssignVariableOp_546" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_547" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" input: "add_449" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_582" op: "ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" input: "^AssignVariableOp_547" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_719/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_719/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_719" op: "Mul" input: "Mul_719/x" input: "Mul_719/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_720/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_720" op: "Mul" input: "Mul_720/x" input: "clip_by_global_norm/clip_by_global_norm/_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_451" op: "AddV2" input: "Mul_719" input: "Mul_720" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_721/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_721/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_721" op: "Mul" input: "Mul_721/x" input: "Mul_721/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_133" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_722/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_722" op: "Mul" input: "Mul_722/x" input: "Square_133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_452" op: "AddV2" input: "Mul_721" input: "Mul_722" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_133" op: "Sqrt" input: "add_452" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_453/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_453" op: "AddV2" input: "Sqrt_133" input: "add_453/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_134" op: "RealDiv" input: "add_451" input: "add_453" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_583" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_723/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_723" op: "Mul" input: "mul_723/x" input: "ReadVariableOp_583" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_454" op: "AddV2" input: "truediv_134" input: "mul_723" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_724" op: "Mul" input: "add" input: "add_454" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_584" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_134" op: "Sub" input: "ReadVariableOp_584" input: "mul_724" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_548" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel" input: "sub_134" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_585" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel" input: "^AssignVariableOp_548" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_549" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" input: "add_451" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_586" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" input: "^AssignVariableOp_549" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_550" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" input: "add_452" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_587" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" input: "^AssignVariableOp_550" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/query/bias/adam_m" } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/query/bias/adam_v" } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/query/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_725/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_725/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_725" op: "Mul" input: "Mul_725/x" input: "Mul_725/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_726/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_726" op: "Mul" input: "Mul_726/x" input: "clip_by_global_norm/clip_by_global_norm/_134" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_455" op: "AddV2" input: "Mul_725" input: "Mul_726" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_727/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_727/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_727" op: "Mul" input: "Mul_727/x" input: "Mul_727/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_134" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_134" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_728/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_728" op: "Mul" input: "Mul_728/x" input: "Square_134" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_456" op: "AddV2" input: "Mul_727" input: "Mul_728" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_134" op: "Sqrt" input: "add_456" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_457/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_457" op: "AddV2" input: "Sqrt_134" input: "add_457/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_135" op: "RealDiv" input: "add_455" input: "add_457" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_729" op: "Mul" input: "add" input: "truediv_135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_588" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_135" op: "Sub" input: "ReadVariableOp_588" input: "mul_729" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_551" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias" input: "sub_135" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_589" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias" input: "^AssignVariableOp_551" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_552" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m" input: "add_455" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_590" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m" input: "^AssignVariableOp_552" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_553" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v" input: "add_456" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_591" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v" input: "^AssignVariableOp_553" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_730/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_730/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_730" op: "Mul" input: "Mul_730/x" input: "Mul_730/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_731/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_731" op: "Mul" input: "Mul_731/x" input: "clip_by_global_norm/clip_by_global_norm/_135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_458" op: "AddV2" input: "Mul_730" input: "Mul_731" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_732/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_732/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_732" op: "Mul" input: "Mul_732/x" input: "Mul_732/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_135" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_733/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_733" op: "Mul" input: "Mul_733/x" input: "Square_135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_459" op: "AddV2" input: "Mul_732" input: "Mul_733" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_135" op: "Sqrt" input: "add_459" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_460/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_460" op: "AddV2" input: "Sqrt_135" input: "add_460/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_136" op: "RealDiv" input: "add_458" input: "add_460" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_592" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_734/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_734" op: "Mul" input: "mul_734/x" input: "ReadVariableOp_592" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_461" op: "AddV2" input: "truediv_136" input: "mul_734" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_735" op: "Mul" input: "add" input: "add_461" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_593" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_136" op: "Sub" input: "ReadVariableOp_593" input: "mul_735" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_554" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel" input: "sub_136" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_594" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel" input: "^AssignVariableOp_554" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_555" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" input: "add_458" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_595" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" input: "^AssignVariableOp_555" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_556" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" input: "add_459" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_596" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" input: "^AssignVariableOp_556" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/key/bias/adam_m" } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/key/bias/adam_v" } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/key/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_736/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_736/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_736" op: "Mul" input: "Mul_736/x" input: "Mul_736/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_737/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_737" op: "Mul" input: "Mul_737/x" input: "clip_by_global_norm/clip_by_global_norm/_136" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_462" op: "AddV2" input: "Mul_736" input: "Mul_737" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_738/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_738/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_738" op: "Mul" input: "Mul_738/x" input: "Mul_738/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_136" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_136" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_739/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_739" op: "Mul" input: "Mul_739/x" input: "Square_136" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_463" op: "AddV2" input: "Mul_738" input: "Mul_739" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_136" op: "Sqrt" input: "add_463" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_464/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_464" op: "AddV2" input: "Sqrt_136" input: "add_464/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_137" op: "RealDiv" input: "add_462" input: "add_464" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_740" op: "Mul" input: "add" input: "truediv_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_597" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_137" op: "Sub" input: "ReadVariableOp_597" input: "mul_740" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_557" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias" input: "sub_137" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_598" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias" input: "^AssignVariableOp_557" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_558" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m" input: "add_462" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_599" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m" input: "^AssignVariableOp_558" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_559" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v" input: "add_463" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_600" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v" input: "^AssignVariableOp_559" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_741/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_741/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_741" op: "Mul" input: "Mul_741/x" input: "Mul_741/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_742/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_742" op: "Mul" input: "Mul_742/x" input: "clip_by_global_norm/clip_by_global_norm/_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_465" op: "AddV2" input: "Mul_741" input: "Mul_742" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_743/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_743/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_743" op: "Mul" input: "Mul_743/x" input: "Mul_743/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_137" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_744/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_744" op: "Mul" input: "Mul_744/x" input: "Square_137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_466" op: "AddV2" input: "Mul_743" input: "Mul_744" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_137" op: "Sqrt" input: "add_466" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_467/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_467" op: "AddV2" input: "Sqrt_137" input: "add_467/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_138" op: "RealDiv" input: "add_465" input: "add_467" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_601" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_745/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_745" op: "Mul" input: "mul_745/x" input: "ReadVariableOp_601" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_468" op: "AddV2" input: "truediv_138" input: "mul_745" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_746" op: "Mul" input: "add" input: "add_468" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_602" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_138" op: "Sub" input: "ReadVariableOp_602" input: "mul_746" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_560" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel" input: "sub_138" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_603" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel" input: "^AssignVariableOp_560" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_561" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" input: "add_465" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_604" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" input: "^AssignVariableOp_561" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_562" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" input: "add_466" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_605" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" input: "^AssignVariableOp_562" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/value/bias/adam_m" } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/self/value/bias/adam_v" } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/self/value/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_747/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_747/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_747" op: "Mul" input: "Mul_747/x" input: "Mul_747/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_748/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_748" op: "Mul" input: "Mul_748/x" input: "clip_by_global_norm/clip_by_global_norm/_138" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_469" op: "AddV2" input: "Mul_747" input: "Mul_748" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_749/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_749/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_749" op: "Mul" input: "Mul_749/x" input: "Mul_749/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_138" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_138" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_750/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_750" op: "Mul" input: "Mul_750/x" input: "Square_138" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_470" op: "AddV2" input: "Mul_749" input: "Mul_750" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_138" op: "Sqrt" input: "add_470" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_471/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_471" op: "AddV2" input: "Sqrt_138" input: "add_471/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_139" op: "RealDiv" input: "add_469" input: "add_471" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_751" op: "Mul" input: "add" input: "truediv_139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_606" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_139" op: "Sub" input: "ReadVariableOp_606" input: "mul_751" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_563" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias" input: "sub_139" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_607" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias" input: "^AssignVariableOp_563" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_564" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m" input: "add_469" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_608" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m" input: "^AssignVariableOp_564" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_565" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v" input: "add_470" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_609" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v" input: "^AssignVariableOp_565" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_752/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_752/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_752" op: "Mul" input: "Mul_752/x" input: "Mul_752/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_753/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_753" op: "Mul" input: "Mul_753/x" input: "clip_by_global_norm/clip_by_global_norm/_139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_472" op: "AddV2" input: "Mul_752" input: "Mul_753" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_754/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_754/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_754" op: "Mul" input: "Mul_754/x" input: "Mul_754/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_139" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_755/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_755" op: "Mul" input: "Mul_755/x" input: "Square_139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_473" op: "AddV2" input: "Mul_754" input: "Mul_755" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_139" op: "Sqrt" input: "add_473" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_474/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_474" op: "AddV2" input: "Sqrt_139" input: "add_474/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_140" op: "RealDiv" input: "add_472" input: "add_474" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_610" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_756/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_756" op: "Mul" input: "mul_756/x" input: "ReadVariableOp_610" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_475" op: "AddV2" input: "truediv_140" input: "mul_756" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_757" op: "Mul" input: "add" input: "add_475" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_611" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_140" op: "Sub" input: "ReadVariableOp_611" input: "mul_757" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_566" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel" input: "sub_140" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_612" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel" input: "^AssignVariableOp_566" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_567" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" input: "add_472" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_613" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" input: "^AssignVariableOp_567" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_568" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" input: "add_473" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_614" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" input: "^AssignVariableOp_568" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_758/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_758/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_758" op: "Mul" input: "Mul_758/x" input: "Mul_758/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_759/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_759" op: "Mul" input: "Mul_759/x" input: "clip_by_global_norm/clip_by_global_norm/_140" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_476" op: "AddV2" input: "Mul_758" input: "Mul_759" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_760/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_760/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_760" op: "Mul" input: "Mul_760/x" input: "Mul_760/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_140" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_140" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_761/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_761" op: "Mul" input: "Mul_761/x" input: "Square_140" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_477" op: "AddV2" input: "Mul_760" input: "Mul_761" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_140" op: "Sqrt" input: "add_477" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_478/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_478" op: "AddV2" input: "Sqrt_140" input: "add_478/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_141" op: "RealDiv" input: "add_476" input: "add_478" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_762" op: "Mul" input: "add" input: "truediv_141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_615" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_141" op: "Sub" input: "ReadVariableOp_615" input: "mul_762" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_569" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias" input: "sub_141" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_616" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias" input: "^AssignVariableOp_569" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_570" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" input: "add_476" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_617" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" input: "^AssignVariableOp_570" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_571" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" input: "add_477" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_618" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" input: "^AssignVariableOp_571" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_763/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_763/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_763" op: "Mul" input: "Mul_763/x" input: "Mul_763/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_764/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_764" op: "Mul" input: "Mul_764/x" input: "clip_by_global_norm/clip_by_global_norm/_141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_479" op: "AddV2" input: "Mul_763" input: "Mul_764" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_765/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_765/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_765" op: "Mul" input: "Mul_765/x" input: "Mul_765/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_141" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_766/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_766" op: "Mul" input: "Mul_766/x" input: "Square_141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_480" op: "AddV2" input: "Mul_765" input: "Mul_766" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_141" op: "Sqrt" input: "add_480" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_481/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_481" op: "AddV2" input: "Sqrt_141" input: "add_481/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_142" op: "RealDiv" input: "add_479" input: "add_481" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_767" op: "Mul" input: "add" input: "truediv_142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_619" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_142" op: "Sub" input: "ReadVariableOp_619" input: "mul_767" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_572" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" input: "sub_142" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_620" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" input: "^AssignVariableOp_572" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_573" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" input: "add_479" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_621" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" input: "^AssignVariableOp_573" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_574" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" input: "add_480" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_622" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" input: "^AssignVariableOp_574" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_768/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_768/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_768" op: "Mul" input: "Mul_768/x" input: "Mul_768/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_769/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_769" op: "Mul" input: "Mul_769/x" input: "clip_by_global_norm/clip_by_global_norm/_142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_482" op: "AddV2" input: "Mul_768" input: "Mul_769" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_770/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_770/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_770" op: "Mul" input: "Mul_770/x" input: "Mul_770/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_142" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_771/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_771" op: "Mul" input: "Mul_771/x" input: "Square_142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_483" op: "AddV2" input: "Mul_770" input: "Mul_771" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_142" op: "Sqrt" input: "add_483" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_484/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_484" op: "AddV2" input: "Sqrt_142" input: "add_484/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_143" op: "RealDiv" input: "add_482" input: "add_484" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_772" op: "Mul" input: "add" input: "truediv_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_623" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_143" op: "Sub" input: "ReadVariableOp_623" input: "mul_772" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_575" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta" input: "sub_143" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_624" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta" input: "^AssignVariableOp_575" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_576" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" input: "add_482" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_625" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" input: "^AssignVariableOp_576" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_577" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" input: "add_483" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_626" op: "ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" input: "^AssignVariableOp_577" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_773/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_773/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_773" op: "Mul" input: "Mul_773/x" input: "Mul_773/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_774/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_774" op: "Mul" input: "Mul_774/x" input: "clip_by_global_norm/clip_by_global_norm/_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_485" op: "AddV2" input: "Mul_773" input: "Mul_774" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_775/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_775/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_775" op: "Mul" input: "Mul_775/x" input: "Mul_775/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_143" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_776/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_776" op: "Mul" input: "Mul_776/x" input: "Square_143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_486" op: "AddV2" input: "Mul_775" input: "Mul_776" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_143" op: "Sqrt" input: "add_486" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_487/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_487" op: "AddV2" input: "Sqrt_143" input: "add_487/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_144" op: "RealDiv" input: "add_485" input: "add_487" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_627" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_777/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_777" op: "Mul" input: "mul_777/x" input: "ReadVariableOp_627" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_488" op: "AddV2" input: "truediv_144" input: "mul_777" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_778" op: "Mul" input: "add" input: "add_488" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_628" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_144" op: "Sub" input: "ReadVariableOp_628" input: "mul_778" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "AssignVariableOp_578" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel" input: "sub_144" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_629" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel" input: "^AssignVariableOp_578" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_579" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" input: "add_485" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_630" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" input: "^AssignVariableOp_579" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_580" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" input: "add_486" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_631" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" input: "^AssignVariableOp_580" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_779/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_779/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_779" op: "Mul" input: "Mul_779/x" input: "Mul_779/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_780/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_780" op: "Mul" input: "Mul_780/x" input: "clip_by_global_norm/clip_by_global_norm/_144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_489" op: "AddV2" input: "Mul_779" input: "Mul_780" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_781/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_781/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_781" op: "Mul" input: "Mul_781/x" input: "Mul_781/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_144" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_782/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_782" op: "Mul" input: "Mul_782/x" input: "Square_144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_490" op: "AddV2" input: "Mul_781" input: "Mul_782" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_144" op: "Sqrt" input: "add_490" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_491/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_491" op: "AddV2" input: "Sqrt_144" input: "add_491/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_145" op: "RealDiv" input: "add_489" input: "add_491" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_783" op: "Mul" input: "add" input: "truediv_145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "ReadVariableOp_632" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_145" op: "Sub" input: "ReadVariableOp_632" input: "mul_783" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "AssignVariableOp_581" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias" input: "sub_145" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_633" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias" input: "^AssignVariableOp_581" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_582" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" input: "add_489" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_634" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" input: "^AssignVariableOp_582" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_583" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" input: "add_490" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_635" op: "ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" input: "^AssignVariableOp_583" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_m" input: "bert/encoder/layer_8/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_v" input: "bert/encoder/layer_8/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_784/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_784/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_784" op: "Mul" input: "Mul_784/x" input: "Mul_784/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_785/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_785" op: "Mul" input: "Mul_785/x" input: "clip_by_global_norm/clip_by_global_norm/_145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_492" op: "AddV2" input: "Mul_784" input: "Mul_785" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_786/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_786/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_786" op: "Mul" input: "Mul_786/x" input: "Mul_786/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_145" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_787/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_787" op: "Mul" input: "Mul_787/x" input: "Square_145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_493" op: "AddV2" input: "Mul_786" input: "Mul_787" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_145" op: "Sqrt" input: "add_493" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_494/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_494" op: "AddV2" input: "Sqrt_145" input: "add_494/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_146" op: "RealDiv" input: "add_492" input: "add_494" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_636" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_788/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_788" op: "Mul" input: "mul_788/x" input: "ReadVariableOp_636" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_495" op: "AddV2" input: "truediv_146" input: "mul_788" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_789" op: "Mul" input: "add" input: "add_495" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_637" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_146" op: "Sub" input: "ReadVariableOp_637" input: "mul_789" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_584" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/kernel" input: "sub_146" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_638" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/kernel" input: "^AssignVariableOp_584" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_585" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_m" input: "add_492" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_639" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_m" input: "^AssignVariableOp_585" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_586" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_v" input: "add_493" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_640" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_v" input: "^AssignVariableOp_586" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/bias/adam_m" input: "bert/encoder/layer_8/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/bias/adam_v" input: "bert/encoder/layer_8/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_790/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_790/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_790" op: "Mul" input: "Mul_790/x" input: "Mul_790/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_791/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_791" op: "Mul" input: "Mul_791/x" input: "clip_by_global_norm/clip_by_global_norm/_146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_496" op: "AddV2" input: "Mul_790" input: "Mul_791" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_792/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_792/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_792" op: "Mul" input: "Mul_792/x" input: "Mul_792/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_146" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_793/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_793" op: "Mul" input: "Mul_793/x" input: "Square_146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_497" op: "AddV2" input: "Mul_792" input: "Mul_793" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_146" op: "Sqrt" input: "add_497" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_498/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_498" op: "AddV2" input: "Sqrt_146" input: "add_498/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_147" op: "RealDiv" input: "add_496" input: "add_498" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_794" op: "Mul" input: "add" input: "truediv_147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_641" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_147" op: "Sub" input: "ReadVariableOp_641" input: "mul_794" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_587" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/bias" input: "sub_147" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_642" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/bias" input: "^AssignVariableOp_587" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_588" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/bias/adam_m" input: "add_496" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_643" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/bias/adam_m" input: "^AssignVariableOp_588" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_589" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/bias/adam_v" input: "add_497" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_644" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/dense/bias/adam_v" input: "^AssignVariableOp_589" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_795/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_795/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_795" op: "Mul" input: "Mul_795/x" input: "Mul_795/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_796/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_796" op: "Mul" input: "Mul_796/x" input: "clip_by_global_norm/clip_by_global_norm/_147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_499" op: "AddV2" input: "Mul_795" input: "Mul_796" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_797/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_797/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_797" op: "Mul" input: "Mul_797/x" input: "Mul_797/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_147" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_798/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_798" op: "Mul" input: "Mul_798/x" input: "Square_147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_500" op: "AddV2" input: "Mul_797" input: "Mul_798" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_147" op: "Sqrt" input: "add_500" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_501/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_501" op: "AddV2" input: "Sqrt_147" input: "add_501/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_148" op: "RealDiv" input: "add_499" input: "add_501" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_799" op: "Mul" input: "add" input: "truediv_148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_645" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_148" op: "Sub" input: "ReadVariableOp_645" input: "mul_799" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_590" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma" input: "sub_148" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_646" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma" input: "^AssignVariableOp_590" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_591" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" input: "add_499" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_647" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" input: "^AssignVariableOp_591" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_592" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" input: "add_500" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_648" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" input: "^AssignVariableOp_592" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_800/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_800/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_800" op: "Mul" input: "Mul_800/x" input: "Mul_800/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_801/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_801" op: "Mul" input: "Mul_801/x" input: "clip_by_global_norm/clip_by_global_norm/_148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_502" op: "AddV2" input: "Mul_800" input: "Mul_801" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_802/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_802/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_802" op: "Mul" input: "Mul_802/x" input: "Mul_802/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_148" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_803/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_803" op: "Mul" input: "Mul_803/x" input: "Square_148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_503" op: "AddV2" input: "Mul_802" input: "Mul_803" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_148" op: "Sqrt" input: "add_503" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_504/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_504" op: "AddV2" input: "Sqrt_148" input: "add_504/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_149" op: "RealDiv" input: "add_502" input: "add_504" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_804" op: "Mul" input: "add" input: "truediv_149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_649" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_149" op: "Sub" input: "ReadVariableOp_649" input: "mul_804" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_593" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta" input: "sub_149" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_650" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta" input: "^AssignVariableOp_593" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_594" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" input: "add_502" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_651" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" input: "^AssignVariableOp_594" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_595" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" input: "add_503" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_652" op: "ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" input: "^AssignVariableOp_595" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_805/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_805/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_805" op: "Mul" input: "Mul_805/x" input: "Mul_805/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_806/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_806" op: "Mul" input: "Mul_806/x" input: "clip_by_global_norm/clip_by_global_norm/_149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_505" op: "AddV2" input: "Mul_805" input: "Mul_806" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_807/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_807/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_807" op: "Mul" input: "Mul_807/x" input: "Mul_807/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_149" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_808/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_808" op: "Mul" input: "Mul_808/x" input: "Square_149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_506" op: "AddV2" input: "Mul_807" input: "Mul_808" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_149" op: "Sqrt" input: "add_506" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_507/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_507" op: "AddV2" input: "Sqrt_149" input: "add_507/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_150" op: "RealDiv" input: "add_505" input: "add_507" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_653" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_809/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_809" op: "Mul" input: "mul_809/x" input: "ReadVariableOp_653" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_508" op: "AddV2" input: "truediv_150" input: "mul_809" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_810" op: "Mul" input: "add" input: "add_508" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_654" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_150" op: "Sub" input: "ReadVariableOp_654" input: "mul_810" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_596" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel" input: "sub_150" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_655" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel" input: "^AssignVariableOp_596" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_597" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" input: "add_505" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_656" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" input: "^AssignVariableOp_597" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_598" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" input: "add_506" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_657" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" input: "^AssignVariableOp_598" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/query/bias/adam_m" } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/query/bias/adam_v" } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/query/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_811/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_811/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_811" op: "Mul" input: "Mul_811/x" input: "Mul_811/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_812/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_812" op: "Mul" input: "Mul_812/x" input: "clip_by_global_norm/clip_by_global_norm/_150" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_509" op: "AddV2" input: "Mul_811" input: "Mul_812" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_813/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_813/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_813" op: "Mul" input: "Mul_813/x" input: "Mul_813/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_150" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_150" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_814/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_814" op: "Mul" input: "Mul_814/x" input: "Square_150" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_510" op: "AddV2" input: "Mul_813" input: "Mul_814" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_150" op: "Sqrt" input: "add_510" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_511/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_511" op: "AddV2" input: "Sqrt_150" input: "add_511/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_151" op: "RealDiv" input: "add_509" input: "add_511" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_815" op: "Mul" input: "add" input: "truediv_151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_658" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_151" op: "Sub" input: "ReadVariableOp_658" input: "mul_815" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_599" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias" input: "sub_151" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_659" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias" input: "^AssignVariableOp_599" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_600" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m" input: "add_509" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_660" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m" input: "^AssignVariableOp_600" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_601" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v" input: "add_510" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_661" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v" input: "^AssignVariableOp_601" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_816/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_816/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_816" op: "Mul" input: "Mul_816/x" input: "Mul_816/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_817/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_817" op: "Mul" input: "Mul_817/x" input: "clip_by_global_norm/clip_by_global_norm/_151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_512" op: "AddV2" input: "Mul_816" input: "Mul_817" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_818/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_818/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_818" op: "Mul" input: "Mul_818/x" input: "Mul_818/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_151" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_819/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_819" op: "Mul" input: "Mul_819/x" input: "Square_151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_513" op: "AddV2" input: "Mul_818" input: "Mul_819" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_151" op: "Sqrt" input: "add_513" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_514/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_514" op: "AddV2" input: "Sqrt_151" input: "add_514/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_152" op: "RealDiv" input: "add_512" input: "add_514" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_662" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_820/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_820" op: "Mul" input: "mul_820/x" input: "ReadVariableOp_662" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_515" op: "AddV2" input: "truediv_152" input: "mul_820" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_821" op: "Mul" input: "add" input: "add_515" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_663" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_152" op: "Sub" input: "ReadVariableOp_663" input: "mul_821" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_602" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel" input: "sub_152" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_664" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel" input: "^AssignVariableOp_602" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_603" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" input: "add_512" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_665" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" input: "^AssignVariableOp_603" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_604" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" input: "add_513" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_666" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" input: "^AssignVariableOp_604" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/key/bias/adam_m" } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/key/bias/adam_v" } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/key/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_822/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_822/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_822" op: "Mul" input: "Mul_822/x" input: "Mul_822/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_823/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_823" op: "Mul" input: "Mul_823/x" input: "clip_by_global_norm/clip_by_global_norm/_152" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_516" op: "AddV2" input: "Mul_822" input: "Mul_823" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_824/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_824/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_824" op: "Mul" input: "Mul_824/x" input: "Mul_824/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_152" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_152" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_825/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_825" op: "Mul" input: "Mul_825/x" input: "Square_152" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_517" op: "AddV2" input: "Mul_824" input: "Mul_825" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_152" op: "Sqrt" input: "add_517" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_518/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_518" op: "AddV2" input: "Sqrt_152" input: "add_518/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_153" op: "RealDiv" input: "add_516" input: "add_518" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_826" op: "Mul" input: "add" input: "truediv_153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_667" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_153" op: "Sub" input: "ReadVariableOp_667" input: "mul_826" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_605" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias" input: "sub_153" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_668" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias" input: "^AssignVariableOp_605" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_606" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m" input: "add_516" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_669" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m" input: "^AssignVariableOp_606" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_607" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v" input: "add_517" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_670" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v" input: "^AssignVariableOp_607" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_827/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_827/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_827" op: "Mul" input: "Mul_827/x" input: "Mul_827/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_828/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_828" op: "Mul" input: "Mul_828/x" input: "clip_by_global_norm/clip_by_global_norm/_153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_519" op: "AddV2" input: "Mul_827" input: "Mul_828" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_829/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_829/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_829" op: "Mul" input: "Mul_829/x" input: "Mul_829/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_153" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_830/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_830" op: "Mul" input: "Mul_830/x" input: "Square_153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_520" op: "AddV2" input: "Mul_829" input: "Mul_830" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_153" op: "Sqrt" input: "add_520" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_521/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_521" op: "AddV2" input: "Sqrt_153" input: "add_521/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_154" op: "RealDiv" input: "add_519" input: "add_521" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_671" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_831/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_831" op: "Mul" input: "mul_831/x" input: "ReadVariableOp_671" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_522" op: "AddV2" input: "truediv_154" input: "mul_831" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_832" op: "Mul" input: "add" input: "add_522" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_672" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_154" op: "Sub" input: "ReadVariableOp_672" input: "mul_832" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_608" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel" input: "sub_154" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_673" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel" input: "^AssignVariableOp_608" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_609" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" input: "add_519" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_674" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" input: "^AssignVariableOp_609" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_610" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" input: "add_520" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_675" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" input: "^AssignVariableOp_610" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/value/bias/adam_m" } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/self/value/bias/adam_v" } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/self/value/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_833/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_833/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_833" op: "Mul" input: "Mul_833/x" input: "Mul_833/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_834/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_834" op: "Mul" input: "Mul_834/x" input: "clip_by_global_norm/clip_by_global_norm/_154" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_523" op: "AddV2" input: "Mul_833" input: "Mul_834" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_835/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_835/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_835" op: "Mul" input: "Mul_835/x" input: "Mul_835/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_154" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_154" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_836/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_836" op: "Mul" input: "Mul_836/x" input: "Square_154" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_524" op: "AddV2" input: "Mul_835" input: "Mul_836" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_154" op: "Sqrt" input: "add_524" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_525/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_525" op: "AddV2" input: "Sqrt_154" input: "add_525/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_155" op: "RealDiv" input: "add_523" input: "add_525" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_837" op: "Mul" input: "add" input: "truediv_155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_676" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_155" op: "Sub" input: "ReadVariableOp_676" input: "mul_837" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_611" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias" input: "sub_155" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_677" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias" input: "^AssignVariableOp_611" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_612" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m" input: "add_523" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_678" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m" input: "^AssignVariableOp_612" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_613" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v" input: "add_524" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_679" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v" input: "^AssignVariableOp_613" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_838/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_838/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_838" op: "Mul" input: "Mul_838/x" input: "Mul_838/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_839/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_839" op: "Mul" input: "Mul_839/x" input: "clip_by_global_norm/clip_by_global_norm/_155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_526" op: "AddV2" input: "Mul_838" input: "Mul_839" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_840/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_840/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_840" op: "Mul" input: "Mul_840/x" input: "Mul_840/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_155" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_841/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_841" op: "Mul" input: "Mul_841/x" input: "Square_155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_527" op: "AddV2" input: "Mul_840" input: "Mul_841" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_155" op: "Sqrt" input: "add_527" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_528/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_528" op: "AddV2" input: "Sqrt_155" input: "add_528/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_156" op: "RealDiv" input: "add_526" input: "add_528" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_680" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_842/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_842" op: "Mul" input: "mul_842/x" input: "ReadVariableOp_680" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_529" op: "AddV2" input: "truediv_156" input: "mul_842" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_843" op: "Mul" input: "add" input: "add_529" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_681" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_156" op: "Sub" input: "ReadVariableOp_681" input: "mul_843" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_614" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel" input: "sub_156" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_682" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel" input: "^AssignVariableOp_614" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_615" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" input: "add_526" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_683" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" input: "^AssignVariableOp_615" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_616" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" input: "add_527" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_684" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" input: "^AssignVariableOp_616" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_844/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_844/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_844" op: "Mul" input: "Mul_844/x" input: "Mul_844/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_845/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_845" op: "Mul" input: "Mul_845/x" input: "clip_by_global_norm/clip_by_global_norm/_156" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_530" op: "AddV2" input: "Mul_844" input: "Mul_845" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_846/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_846/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_846" op: "Mul" input: "Mul_846/x" input: "Mul_846/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_156" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_156" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_847/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_847" op: "Mul" input: "Mul_847/x" input: "Square_156" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_531" op: "AddV2" input: "Mul_846" input: "Mul_847" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_156" op: "Sqrt" input: "add_531" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_532/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_532" op: "AddV2" input: "Sqrt_156" input: "add_532/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_157" op: "RealDiv" input: "add_530" input: "add_532" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_848" op: "Mul" input: "add" input: "truediv_157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_685" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_157" op: "Sub" input: "ReadVariableOp_685" input: "mul_848" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_617" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias" input: "sub_157" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_686" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias" input: "^AssignVariableOp_617" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_618" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" input: "add_530" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_687" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" input: "^AssignVariableOp_618" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_619" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" input: "add_531" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_688" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" input: "^AssignVariableOp_619" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_849/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_849/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_849" op: "Mul" input: "Mul_849/x" input: "Mul_849/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_850/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_850" op: "Mul" input: "Mul_850/x" input: "clip_by_global_norm/clip_by_global_norm/_157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_533" op: "AddV2" input: "Mul_849" input: "Mul_850" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_851/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_851/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_851" op: "Mul" input: "Mul_851/x" input: "Mul_851/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_157" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_852/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_852" op: "Mul" input: "Mul_852/x" input: "Square_157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_534" op: "AddV2" input: "Mul_851" input: "Mul_852" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_157" op: "Sqrt" input: "add_534" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_535/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_535" op: "AddV2" input: "Sqrt_157" input: "add_535/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_158" op: "RealDiv" input: "add_533" input: "add_535" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_853" op: "Mul" input: "add" input: "truediv_158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_689" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_158" op: "Sub" input: "ReadVariableOp_689" input: "mul_853" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_620" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" input: "sub_158" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_690" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" input: "^AssignVariableOp_620" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_621" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" input: "add_533" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_691" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" input: "^AssignVariableOp_621" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_622" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" input: "add_534" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_692" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" input: "^AssignVariableOp_622" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_854/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_854/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_854" op: "Mul" input: "Mul_854/x" input: "Mul_854/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_855/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_855" op: "Mul" input: "Mul_855/x" input: "clip_by_global_norm/clip_by_global_norm/_158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_536" op: "AddV2" input: "Mul_854" input: "Mul_855" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_856/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_856/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_856" op: "Mul" input: "Mul_856/x" input: "Mul_856/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_158" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_857/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_857" op: "Mul" input: "Mul_857/x" input: "Square_158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_537" op: "AddV2" input: "Mul_856" input: "Mul_857" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_158" op: "Sqrt" input: "add_537" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_538/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_538" op: "AddV2" input: "Sqrt_158" input: "add_538/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_159" op: "RealDiv" input: "add_536" input: "add_538" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_858" op: "Mul" input: "add" input: "truediv_159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_693" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_159" op: "Sub" input: "ReadVariableOp_693" input: "mul_858" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_623" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta" input: "sub_159" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_694" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta" input: "^AssignVariableOp_623" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_624" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" input: "add_536" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_695" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" input: "^AssignVariableOp_624" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_625" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" input: "add_537" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_696" op: "ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" input: "^AssignVariableOp_625" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_859/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_859/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_859" op: "Mul" input: "Mul_859/x" input: "Mul_859/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_860/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_860" op: "Mul" input: "Mul_860/x" input: "clip_by_global_norm/clip_by_global_norm/_159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_539" op: "AddV2" input: "Mul_859" input: "Mul_860" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_861/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_861/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_861" op: "Mul" input: "Mul_861/x" input: "Mul_861/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_159" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_862/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_862" op: "Mul" input: "Mul_862/x" input: "Square_159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_540" op: "AddV2" input: "Mul_861" input: "Mul_862" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_159" op: "Sqrt" input: "add_540" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_541/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_541" op: "AddV2" input: "Sqrt_159" input: "add_541/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_160" op: "RealDiv" input: "add_539" input: "add_541" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_697" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_863/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_863" op: "Mul" input: "mul_863/x" input: "ReadVariableOp_697" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_542" op: "AddV2" input: "truediv_160" input: "mul_863" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_864" op: "Mul" input: "add" input: "add_542" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_698" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_160" op: "Sub" input: "ReadVariableOp_698" input: "mul_864" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "AssignVariableOp_626" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel" input: "sub_160" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_699" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel" input: "^AssignVariableOp_626" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_627" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" input: "add_539" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_700" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" input: "^AssignVariableOp_627" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_628" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" input: "add_540" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_701" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" input: "^AssignVariableOp_628" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_865/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_865/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_865" op: "Mul" input: "Mul_865/x" input: "Mul_865/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_866/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_866" op: "Mul" input: "Mul_866/x" input: "clip_by_global_norm/clip_by_global_norm/_160" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_543" op: "AddV2" input: "Mul_865" input: "Mul_866" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_867/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_867/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_867" op: "Mul" input: "Mul_867/x" input: "Mul_867/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_160" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_160" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_868/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_868" op: "Mul" input: "Mul_868/x" input: "Square_160" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_544" op: "AddV2" input: "Mul_867" input: "Mul_868" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_160" op: "Sqrt" input: "add_544" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_545/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_545" op: "AddV2" input: "Sqrt_160" input: "add_545/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_161" op: "RealDiv" input: "add_543" input: "add_545" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_869" op: "Mul" input: "add" input: "truediv_161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "ReadVariableOp_702" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_161" op: "Sub" input: "ReadVariableOp_702" input: "mul_869" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "AssignVariableOp_629" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias" input: "sub_161" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_703" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias" input: "^AssignVariableOp_629" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_630" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" input: "add_543" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_704" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" input: "^AssignVariableOp_630" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_631" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" input: "add_544" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_705" op: "ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" input: "^AssignVariableOp_631" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_m" input: "bert/encoder/layer_9/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_v" input: "bert/encoder/layer_9/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_870/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_870/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_870" op: "Mul" input: "Mul_870/x" input: "Mul_870/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_871/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_871" op: "Mul" input: "Mul_871/x" input: "clip_by_global_norm/clip_by_global_norm/_161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_546" op: "AddV2" input: "Mul_870" input: "Mul_871" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_872/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_872/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_872" op: "Mul" input: "Mul_872/x" input: "Mul_872/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_161" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_873/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_873" op: "Mul" input: "Mul_873/x" input: "Square_161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_547" op: "AddV2" input: "Mul_872" input: "Mul_873" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_161" op: "Sqrt" input: "add_547" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_548/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_548" op: "AddV2" input: "Sqrt_161" input: "add_548/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_162" op: "RealDiv" input: "add_546" input: "add_548" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_706" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_874/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_874" op: "Mul" input: "mul_874/x" input: "ReadVariableOp_706" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_549" op: "AddV2" input: "truediv_162" input: "mul_874" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_875" op: "Mul" input: "add" input: "add_549" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_707" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_162" op: "Sub" input: "ReadVariableOp_707" input: "mul_875" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_632" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/kernel" input: "sub_162" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_708" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/kernel" input: "^AssignVariableOp_632" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_633" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_m" input: "add_546" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_709" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_m" input: "^AssignVariableOp_633" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_634" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_v" input: "add_547" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_710" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_v" input: "^AssignVariableOp_634" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/bias/adam_m" input: "bert/encoder/layer_9/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/bias/adam_v" input: "bert/encoder/layer_9/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_876/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_876/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_876" op: "Mul" input: "Mul_876/x" input: "Mul_876/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_877/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_877" op: "Mul" input: "Mul_877/x" input: "clip_by_global_norm/clip_by_global_norm/_162" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_550" op: "AddV2" input: "Mul_876" input: "Mul_877" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_878/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_878/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_878" op: "Mul" input: "Mul_878/x" input: "Mul_878/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_162" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_162" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_879/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_879" op: "Mul" input: "Mul_879/x" input: "Square_162" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_551" op: "AddV2" input: "Mul_878" input: "Mul_879" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_162" op: "Sqrt" input: "add_551" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_552/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_552" op: "AddV2" input: "Sqrt_162" input: "add_552/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_163" op: "RealDiv" input: "add_550" input: "add_552" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_880" op: "Mul" input: "add" input: "truediv_163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_711" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_163" op: "Sub" input: "ReadVariableOp_711" input: "mul_880" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_635" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/bias" input: "sub_163" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_712" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/bias" input: "^AssignVariableOp_635" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_636" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/bias/adam_m" input: "add_550" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_713" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/bias/adam_m" input: "^AssignVariableOp_636" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_637" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/bias/adam_v" input: "add_551" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_714" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/dense/bias/adam_v" input: "^AssignVariableOp_637" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_881/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_881/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_881" op: "Mul" input: "Mul_881/x" input: "Mul_881/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_882/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_882" op: "Mul" input: "Mul_882/x" input: "clip_by_global_norm/clip_by_global_norm/_163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_553" op: "AddV2" input: "Mul_881" input: "Mul_882" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_883/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_883/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_883" op: "Mul" input: "Mul_883/x" input: "Mul_883/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_163" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_884/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_884" op: "Mul" input: "Mul_884/x" input: "Square_163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_554" op: "AddV2" input: "Mul_883" input: "Mul_884" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_163" op: "Sqrt" input: "add_554" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_555/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_555" op: "AddV2" input: "Sqrt_163" input: "add_555/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_164" op: "RealDiv" input: "add_553" input: "add_555" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_885" op: "Mul" input: "add" input: "truediv_164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_715" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_164" op: "Sub" input: "ReadVariableOp_715" input: "mul_885" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_638" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma" input: "sub_164" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_716" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma" input: "^AssignVariableOp_638" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_639" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" input: "add_553" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_717" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" input: "^AssignVariableOp_639" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_640" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" input: "add_554" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_718" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" input: "^AssignVariableOp_640" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_886/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_886/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_886" op: "Mul" input: "Mul_886/x" input: "Mul_886/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_887/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_887" op: "Mul" input: "Mul_887/x" input: "clip_by_global_norm/clip_by_global_norm/_164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_556" op: "AddV2" input: "Mul_886" input: "Mul_887" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_888/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_888/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_888" op: "Mul" input: "Mul_888/x" input: "Mul_888/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_164" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_889/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_889" op: "Mul" input: "Mul_889/x" input: "Square_164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_557" op: "AddV2" input: "Mul_888" input: "Mul_889" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_164" op: "Sqrt" input: "add_557" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_558/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_558" op: "AddV2" input: "Sqrt_164" input: "add_558/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_165" op: "RealDiv" input: "add_556" input: "add_558" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_890" op: "Mul" input: "add" input: "truediv_165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_719" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_165" op: "Sub" input: "ReadVariableOp_719" input: "mul_890" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_641" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta" input: "sub_165" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_720" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta" input: "^AssignVariableOp_641" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_642" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" input: "add_556" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_721" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" input: "^AssignVariableOp_642" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_643" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" input: "add_557" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_722" op: "ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" input: "^AssignVariableOp_643" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_891/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_891/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_891" op: "Mul" input: "Mul_891/x" input: "Mul_891/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_892/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_892" op: "Mul" input: "Mul_892/x" input: "clip_by_global_norm/clip_by_global_norm/_165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_559" op: "AddV2" input: "Mul_891" input: "Mul_892" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_893/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_893/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_893" op: "Mul" input: "Mul_893/x" input: "Mul_893/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_165" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_894/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_894" op: "Mul" input: "Mul_894/x" input: "Square_165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_560" op: "AddV2" input: "Mul_893" input: "Mul_894" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_165" op: "Sqrt" input: "add_560" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_561/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_561" op: "AddV2" input: "Sqrt_165" input: "add_561/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_166" op: "RealDiv" input: "add_559" input: "add_561" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_723" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_895/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_895" op: "Mul" input: "mul_895/x" input: "ReadVariableOp_723" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_562" op: "AddV2" input: "truediv_166" input: "mul_895" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_896" op: "Mul" input: "add" input: "add_562" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_724" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_166" op: "Sub" input: "ReadVariableOp_724" input: "mul_896" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_644" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel" input: "sub_166" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_725" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel" input: "^AssignVariableOp_644" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_645" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" input: "add_559" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_726" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" input: "^AssignVariableOp_645" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_646" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" input: "add_560" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_727" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" input: "^AssignVariableOp_646" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/query/bias/adam_m" } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/query/bias/adam_v" } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/query/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_897/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_897/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_897" op: "Mul" input: "Mul_897/x" input: "Mul_897/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_898/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_898" op: "Mul" input: "Mul_898/x" input: "clip_by_global_norm/clip_by_global_norm/_166" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_563" op: "AddV2" input: "Mul_897" input: "Mul_898" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_899/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_899/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_899" op: "Mul" input: "Mul_899/x" input: "Mul_899/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_166" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_166" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_900/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_900" op: "Mul" input: "Mul_900/x" input: "Square_166" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_564" op: "AddV2" input: "Mul_899" input: "Mul_900" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_166" op: "Sqrt" input: "add_564" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_565/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_565" op: "AddV2" input: "Sqrt_166" input: "add_565/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_167" op: "RealDiv" input: "add_563" input: "add_565" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_901" op: "Mul" input: "add" input: "truediv_167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_728" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_167" op: "Sub" input: "ReadVariableOp_728" input: "mul_901" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_647" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias" input: "sub_167" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_729" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias" input: "^AssignVariableOp_647" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_648" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m" input: "add_563" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_730" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m" input: "^AssignVariableOp_648" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_649" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v" input: "add_564" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_731" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v" input: "^AssignVariableOp_649" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_902/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_902/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_902" op: "Mul" input: "Mul_902/x" input: "Mul_902/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_903/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_903" op: "Mul" input: "Mul_903/x" input: "clip_by_global_norm/clip_by_global_norm/_167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_566" op: "AddV2" input: "Mul_902" input: "Mul_903" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_904/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_904/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_904" op: "Mul" input: "Mul_904/x" input: "Mul_904/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_167" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_905/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_905" op: "Mul" input: "Mul_905/x" input: "Square_167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_567" op: "AddV2" input: "Mul_904" input: "Mul_905" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_167" op: "Sqrt" input: "add_567" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_568/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_568" op: "AddV2" input: "Sqrt_167" input: "add_568/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_168" op: "RealDiv" input: "add_566" input: "add_568" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_732" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_906/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_906" op: "Mul" input: "mul_906/x" input: "ReadVariableOp_732" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_569" op: "AddV2" input: "truediv_168" input: "mul_906" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_907" op: "Mul" input: "add" input: "add_569" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_733" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_168" op: "Sub" input: "ReadVariableOp_733" input: "mul_907" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_650" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel" input: "sub_168" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_734" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel" input: "^AssignVariableOp_650" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_651" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" input: "add_566" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_735" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" input: "^AssignVariableOp_651" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_652" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" input: "add_567" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_736" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" input: "^AssignVariableOp_652" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/key/bias/adam_m" } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/key/bias/adam_v" } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/key/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_908/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_908/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_908" op: "Mul" input: "Mul_908/x" input: "Mul_908/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_909/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_909" op: "Mul" input: "Mul_909/x" input: "clip_by_global_norm/clip_by_global_norm/_168" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_570" op: "AddV2" input: "Mul_908" input: "Mul_909" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_910/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_910/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_910" op: "Mul" input: "Mul_910/x" input: "Mul_910/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_168" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_168" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_911/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_911" op: "Mul" input: "Mul_911/x" input: "Square_168" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_571" op: "AddV2" input: "Mul_910" input: "Mul_911" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_168" op: "Sqrt" input: "add_571" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_572/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_572" op: "AddV2" input: "Sqrt_168" input: "add_572/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_169" op: "RealDiv" input: "add_570" input: "add_572" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_912" op: "Mul" input: "add" input: "truediv_169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_737" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_169" op: "Sub" input: "ReadVariableOp_737" input: "mul_912" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_653" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias" input: "sub_169" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_738" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias" input: "^AssignVariableOp_653" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_654" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m" input: "add_570" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_739" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m" input: "^AssignVariableOp_654" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_655" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v" input: "add_571" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_740" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v" input: "^AssignVariableOp_655" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_913/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_913/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_913" op: "Mul" input: "Mul_913/x" input: "Mul_913/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_914/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_914" op: "Mul" input: "Mul_914/x" input: "clip_by_global_norm/clip_by_global_norm/_169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_573" op: "AddV2" input: "Mul_913" input: "Mul_914" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_915/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_915/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_915" op: "Mul" input: "Mul_915/x" input: "Mul_915/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_169" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_916/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_916" op: "Mul" input: "Mul_916/x" input: "Square_169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_574" op: "AddV2" input: "Mul_915" input: "Mul_916" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_169" op: "Sqrt" input: "add_574" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_575/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_575" op: "AddV2" input: "Sqrt_169" input: "add_575/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_170" op: "RealDiv" input: "add_573" input: "add_575" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_741" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_917/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_917" op: "Mul" input: "mul_917/x" input: "ReadVariableOp_741" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_576" op: "AddV2" input: "truediv_170" input: "mul_917" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_918" op: "Mul" input: "add" input: "add_576" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_742" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_170" op: "Sub" input: "ReadVariableOp_742" input: "mul_918" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_656" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel" input: "sub_170" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_743" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel" input: "^AssignVariableOp_656" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_657" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" input: "add_573" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_744" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" input: "^AssignVariableOp_657" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_658" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" input: "add_574" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_745" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" input: "^AssignVariableOp_658" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/value/bias/adam_m" } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/self/value/bias/adam_v" } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/self/value/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_919/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_919/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_919" op: "Mul" input: "Mul_919/x" input: "Mul_919/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_920/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_920" op: "Mul" input: "Mul_920/x" input: "clip_by_global_norm/clip_by_global_norm/_170" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_577" op: "AddV2" input: "Mul_919" input: "Mul_920" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_921/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_921/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_921" op: "Mul" input: "Mul_921/x" input: "Mul_921/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_170" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_170" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_922/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_922" op: "Mul" input: "Mul_922/x" input: "Square_170" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_578" op: "AddV2" input: "Mul_921" input: "Mul_922" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_170" op: "Sqrt" input: "add_578" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_579/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_579" op: "AddV2" input: "Sqrt_170" input: "add_579/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_171" op: "RealDiv" input: "add_577" input: "add_579" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_923" op: "Mul" input: "add" input: "truediv_171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_746" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_171" op: "Sub" input: "ReadVariableOp_746" input: "mul_923" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_659" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias" input: "sub_171" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_747" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias" input: "^AssignVariableOp_659" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_660" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m" input: "add_577" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_748" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m" input: "^AssignVariableOp_660" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_661" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v" input: "add_578" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_749" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v" input: "^AssignVariableOp_661" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_924/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_924/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_924" op: "Mul" input: "Mul_924/x" input: "Mul_924/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_925/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_925" op: "Mul" input: "Mul_925/x" input: "clip_by_global_norm/clip_by_global_norm/_171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_580" op: "AddV2" input: "Mul_924" input: "Mul_925" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_926/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_926/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_926" op: "Mul" input: "Mul_926/x" input: "Mul_926/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_171" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_927/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_927" op: "Mul" input: "Mul_927/x" input: "Square_171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_581" op: "AddV2" input: "Mul_926" input: "Mul_927" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_171" op: "Sqrt" input: "add_581" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_582/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_582" op: "AddV2" input: "Sqrt_171" input: "add_582/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_172" op: "RealDiv" input: "add_580" input: "add_582" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_750" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_928/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_928" op: "Mul" input: "mul_928/x" input: "ReadVariableOp_750" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_583" op: "AddV2" input: "truediv_172" input: "mul_928" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_929" op: "Mul" input: "add" input: "add_583" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_751" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_172" op: "Sub" input: "ReadVariableOp_751" input: "mul_929" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_662" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel" input: "sub_172" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_752" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel" input: "^AssignVariableOp_662" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_663" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" input: "add_580" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_753" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" input: "^AssignVariableOp_663" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_664" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" input: "add_581" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_754" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" input: "^AssignVariableOp_664" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_930/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_930/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_930" op: "Mul" input: "Mul_930/x" input: "Mul_930/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_931/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_931" op: "Mul" input: "Mul_931/x" input: "clip_by_global_norm/clip_by_global_norm/_172" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_584" op: "AddV2" input: "Mul_930" input: "Mul_931" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_932/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_932/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_932" op: "Mul" input: "Mul_932/x" input: "Mul_932/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_172" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_172" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_933/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_933" op: "Mul" input: "Mul_933/x" input: "Square_172" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_585" op: "AddV2" input: "Mul_932" input: "Mul_933" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_172" op: "Sqrt" input: "add_585" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_586/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_586" op: "AddV2" input: "Sqrt_172" input: "add_586/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_173" op: "RealDiv" input: "add_584" input: "add_586" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_934" op: "Mul" input: "add" input: "truediv_173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_755" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_173" op: "Sub" input: "ReadVariableOp_755" input: "mul_934" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_665" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias" input: "sub_173" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_756" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias" input: "^AssignVariableOp_665" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_666" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" input: "add_584" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_757" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" input: "^AssignVariableOp_666" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_667" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" input: "add_585" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_758" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" input: "^AssignVariableOp_667" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_935/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_935/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_935" op: "Mul" input: "Mul_935/x" input: "Mul_935/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_936/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_936" op: "Mul" input: "Mul_936/x" input: "clip_by_global_norm/clip_by_global_norm/_173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_587" op: "AddV2" input: "Mul_935" input: "Mul_936" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_937/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_937/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_937" op: "Mul" input: "Mul_937/x" input: "Mul_937/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_173" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_938/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_938" op: "Mul" input: "Mul_938/x" input: "Square_173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_588" op: "AddV2" input: "Mul_937" input: "Mul_938" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_173" op: "Sqrt" input: "add_588" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_589/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_589" op: "AddV2" input: "Sqrt_173" input: "add_589/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_174" op: "RealDiv" input: "add_587" input: "add_589" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_939" op: "Mul" input: "add" input: "truediv_174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_759" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_174" op: "Sub" input: "ReadVariableOp_759" input: "mul_939" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_668" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" input: "sub_174" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_760" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" input: "^AssignVariableOp_668" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_669" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" input: "add_587" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_761" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" input: "^AssignVariableOp_669" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_670" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" input: "add_588" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_762" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" input: "^AssignVariableOp_670" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_940/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_940/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_940" op: "Mul" input: "Mul_940/x" input: "Mul_940/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_941/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_941" op: "Mul" input: "Mul_941/x" input: "clip_by_global_norm/clip_by_global_norm/_174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_590" op: "AddV2" input: "Mul_940" input: "Mul_941" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_942/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_942/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_942" op: "Mul" input: "Mul_942/x" input: "Mul_942/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_174" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_943/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_943" op: "Mul" input: "Mul_943/x" input: "Square_174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_591" op: "AddV2" input: "Mul_942" input: "Mul_943" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_174" op: "Sqrt" input: "add_591" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_592/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_592" op: "AddV2" input: "Sqrt_174" input: "add_592/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_175" op: "RealDiv" input: "add_590" input: "add_592" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_944" op: "Mul" input: "add" input: "truediv_175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_763" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_175" op: "Sub" input: "ReadVariableOp_763" input: "mul_944" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_671" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta" input: "sub_175" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_764" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta" input: "^AssignVariableOp_671" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_672" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" input: "add_590" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_765" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" input: "^AssignVariableOp_672" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_673" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" input: "add_591" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_766" op: "ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" input: "^AssignVariableOp_673" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_945/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_945/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_945" op: "Mul" input: "Mul_945/x" input: "Mul_945/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_946/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_946" op: "Mul" input: "Mul_946/x" input: "clip_by_global_norm/clip_by_global_norm/_175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_593" op: "AddV2" input: "Mul_945" input: "Mul_946" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_947/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_947/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_947" op: "Mul" input: "Mul_947/x" input: "Mul_947/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_175" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_948/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_948" op: "Mul" input: "Mul_948/x" input: "Square_175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_594" op: "AddV2" input: "Mul_947" input: "Mul_948" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_175" op: "Sqrt" input: "add_594" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_595/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_595" op: "AddV2" input: "Sqrt_175" input: "add_595/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_176" op: "RealDiv" input: "add_593" input: "add_595" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_767" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_949/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_949" op: "Mul" input: "mul_949/x" input: "ReadVariableOp_767" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_596" op: "AddV2" input: "truediv_176" input: "mul_949" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_950" op: "Mul" input: "add" input: "add_596" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_768" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_176" op: "Sub" input: "ReadVariableOp_768" input: "mul_950" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "AssignVariableOp_674" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel" input: "sub_176" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_769" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel" input: "^AssignVariableOp_674" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_675" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" input: "add_593" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_770" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" input: "^AssignVariableOp_675" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_676" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" input: "add_594" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_771" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" input: "^AssignVariableOp_676" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_951/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_951/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_951" op: "Mul" input: "Mul_951/x" input: "Mul_951/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_952/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_952" op: "Mul" input: "Mul_952/x" input: "clip_by_global_norm/clip_by_global_norm/_176" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_597" op: "AddV2" input: "Mul_951" input: "Mul_952" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_953/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_953/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_953" op: "Mul" input: "Mul_953/x" input: "Mul_953/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_176" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_176" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_954/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_954" op: "Mul" input: "Mul_954/x" input: "Square_176" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_598" op: "AddV2" input: "Mul_953" input: "Mul_954" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_176" op: "Sqrt" input: "add_598" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_599/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_599" op: "AddV2" input: "Sqrt_176" input: "add_599/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_177" op: "RealDiv" input: "add_597" input: "add_599" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_955" op: "Mul" input: "add" input: "truediv_177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "ReadVariableOp_772" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_177" op: "Sub" input: "ReadVariableOp_772" input: "mul_955" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "AssignVariableOp_677" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias" input: "sub_177" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_773" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias" input: "^AssignVariableOp_677" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_678" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" input: "add_597" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_774" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" input: "^AssignVariableOp_678" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_679" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" input: "add_598" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_775" op: "ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" input: "^AssignVariableOp_679" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_m" input: "bert/encoder/layer_10/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_v" input: "bert/encoder/layer_10/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_956/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_956/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_956" op: "Mul" input: "Mul_956/x" input: "Mul_956/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_957/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_957" op: "Mul" input: "Mul_957/x" input: "clip_by_global_norm/clip_by_global_norm/_177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_600" op: "AddV2" input: "Mul_956" input: "Mul_957" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_958/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_958/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_958" op: "Mul" input: "Mul_958/x" input: "Mul_958/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_177" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_959/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_959" op: "Mul" input: "Mul_959/x" input: "Square_177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_601" op: "AddV2" input: "Mul_958" input: "Mul_959" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_177" op: "Sqrt" input: "add_601" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_602/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_602" op: "AddV2" input: "Sqrt_177" input: "add_602/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_178" op: "RealDiv" input: "add_600" input: "add_602" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_776" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_960/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_960" op: "Mul" input: "mul_960/x" input: "ReadVariableOp_776" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_603" op: "AddV2" input: "truediv_178" input: "mul_960" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_961" op: "Mul" input: "add" input: "add_603" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_777" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_178" op: "Sub" input: "ReadVariableOp_777" input: "mul_961" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_680" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/kernel" input: "sub_178" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_778" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/kernel" input: "^AssignVariableOp_680" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_681" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_m" input: "add_600" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_779" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_m" input: "^AssignVariableOp_681" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_682" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_v" input: "add_601" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_780" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_v" input: "^AssignVariableOp_682" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/bias/adam_m" input: "bert/encoder/layer_10/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/bias/adam_v" input: "bert/encoder/layer_10/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_962/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_962/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_962" op: "Mul" input: "Mul_962/x" input: "Mul_962/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_963/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_963" op: "Mul" input: "Mul_963/x" input: "clip_by_global_norm/clip_by_global_norm/_178" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_604" op: "AddV2" input: "Mul_962" input: "Mul_963" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_964/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_964/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_964" op: "Mul" input: "Mul_964/x" input: "Mul_964/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_178" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_178" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_965/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_965" op: "Mul" input: "Mul_965/x" input: "Square_178" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_605" op: "AddV2" input: "Mul_964" input: "Mul_965" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_178" op: "Sqrt" input: "add_605" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_606/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_606" op: "AddV2" input: "Sqrt_178" input: "add_606/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_179" op: "RealDiv" input: "add_604" input: "add_606" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_966" op: "Mul" input: "add" input: "truediv_179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_781" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_179" op: "Sub" input: "ReadVariableOp_781" input: "mul_966" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_683" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/bias" input: "sub_179" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_782" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/bias" input: "^AssignVariableOp_683" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_684" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/bias/adam_m" input: "add_604" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_783" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/bias/adam_m" input: "^AssignVariableOp_684" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_685" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/bias/adam_v" input: "add_605" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_784" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/dense/bias/adam_v" input: "^AssignVariableOp_685" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_967/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_967/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_967" op: "Mul" input: "Mul_967/x" input: "Mul_967/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_968/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_968" op: "Mul" input: "Mul_968/x" input: "clip_by_global_norm/clip_by_global_norm/_179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_607" op: "AddV2" input: "Mul_967" input: "Mul_968" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_969/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_969/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_969" op: "Mul" input: "Mul_969/x" input: "Mul_969/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_179" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_970/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_970" op: "Mul" input: "Mul_970/x" input: "Square_179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_608" op: "AddV2" input: "Mul_969" input: "Mul_970" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_179" op: "Sqrt" input: "add_608" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_609/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_609" op: "AddV2" input: "Sqrt_179" input: "add_609/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_180" op: "RealDiv" input: "add_607" input: "add_609" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_971" op: "Mul" input: "add" input: "truediv_180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_785" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_180" op: "Sub" input: "ReadVariableOp_785" input: "mul_971" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_686" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma" input: "sub_180" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_786" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma" input: "^AssignVariableOp_686" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_687" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" input: "add_607" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_787" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" input: "^AssignVariableOp_687" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_688" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" input: "add_608" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_788" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" input: "^AssignVariableOp_688" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_972/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_972/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_972" op: "Mul" input: "Mul_972/x" input: "Mul_972/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_973/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_973" op: "Mul" input: "Mul_973/x" input: "clip_by_global_norm/clip_by_global_norm/_180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_610" op: "AddV2" input: "Mul_972" input: "Mul_973" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_974/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_974/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_974" op: "Mul" input: "Mul_974/x" input: "Mul_974/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_180" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_975/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_975" op: "Mul" input: "Mul_975/x" input: "Square_180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_611" op: "AddV2" input: "Mul_974" input: "Mul_975" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_180" op: "Sqrt" input: "add_611" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_612/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_612" op: "AddV2" input: "Sqrt_180" input: "add_612/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_181" op: "RealDiv" input: "add_610" input: "add_612" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_976" op: "Mul" input: "add" input: "truediv_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_789" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_181" op: "Sub" input: "ReadVariableOp_789" input: "mul_976" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_689" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta" input: "sub_181" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_790" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta" input: "^AssignVariableOp_689" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_690" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" input: "add_610" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_791" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" input: "^AssignVariableOp_690" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_691" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" input: "add_611" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_792" op: "ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" input: "^AssignVariableOp_691" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_977/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_977/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_977" op: "Mul" input: "Mul_977/x" input: "Mul_977/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_978/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_978" op: "Mul" input: "Mul_978/x" input: "clip_by_global_norm/clip_by_global_norm/_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_613" op: "AddV2" input: "Mul_977" input: "Mul_978" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_979/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_979/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_979" op: "Mul" input: "Mul_979/x" input: "Mul_979/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_181" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_980/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_980" op: "Mul" input: "Mul_980/x" input: "Square_181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_614" op: "AddV2" input: "Mul_979" input: "Mul_980" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_181" op: "Sqrt" input: "add_614" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_615/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_615" op: "AddV2" input: "Sqrt_181" input: "add_615/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_182" op: "RealDiv" input: "add_613" input: "add_615" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_793" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_981/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_981" op: "Mul" input: "mul_981/x" input: "ReadVariableOp_793" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_616" op: "AddV2" input: "truediv_182" input: "mul_981" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_982" op: "Mul" input: "add" input: "add_616" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_794" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_182" op: "Sub" input: "ReadVariableOp_794" input: "mul_982" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_692" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel" input: "sub_182" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_795" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel" input: "^AssignVariableOp_692" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_693" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" input: "add_613" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_796" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" input: "^AssignVariableOp_693" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_694" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" input: "add_614" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_797" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" input: "^AssignVariableOp_694" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/query/bias/adam_m" } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/query/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/query/bias/adam_v" } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/query/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_983/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_983/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_983" op: "Mul" input: "Mul_983/x" input: "Mul_983/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_984/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_984" op: "Mul" input: "Mul_984/x" input: "clip_by_global_norm/clip_by_global_norm/_182" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_617" op: "AddV2" input: "Mul_983" input: "Mul_984" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_985/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_985/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_985" op: "Mul" input: "Mul_985/x" input: "Mul_985/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_182" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_182" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_986/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_986" op: "Mul" input: "Mul_986/x" input: "Square_182" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_618" op: "AddV2" input: "Mul_985" input: "Mul_986" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_182" op: "Sqrt" input: "add_618" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_619/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_619" op: "AddV2" input: "Sqrt_182" input: "add_619/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_183" op: "RealDiv" input: "add_617" input: "add_619" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_987" op: "Mul" input: "add" input: "truediv_183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_798" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_183" op: "Sub" input: "ReadVariableOp_798" input: "mul_987" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_695" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias" input: "sub_183" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_799" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias" input: "^AssignVariableOp_695" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_696" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m" input: "add_617" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_800" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m" input: "^AssignVariableOp_696" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_697" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v" input: "add_618" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_801" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v" input: "^AssignVariableOp_697" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_988/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_988/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_988" op: "Mul" input: "Mul_988/x" input: "Mul_988/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_989/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_989" op: "Mul" input: "Mul_989/x" input: "clip_by_global_norm/clip_by_global_norm/_183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_620" op: "AddV2" input: "Mul_988" input: "Mul_989" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_990/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_990/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_990" op: "Mul" input: "Mul_990/x" input: "Mul_990/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_183" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_991/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_991" op: "Mul" input: "Mul_991/x" input: "Square_183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_621" op: "AddV2" input: "Mul_990" input: "Mul_991" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_183" op: "Sqrt" input: "add_621" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_622/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_622" op: "AddV2" input: "Sqrt_183" input: "add_622/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_184" op: "RealDiv" input: "add_620" input: "add_622" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_802" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_992/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_992" op: "Mul" input: "mul_992/x" input: "ReadVariableOp_802" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_623" op: "AddV2" input: "truediv_184" input: "mul_992" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_993" op: "Mul" input: "add" input: "add_623" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_803" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_184" op: "Sub" input: "ReadVariableOp_803" input: "mul_993" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_698" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel" input: "sub_184" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_804" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel" input: "^AssignVariableOp_698" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_699" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" input: "add_620" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_805" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" input: "^AssignVariableOp_699" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_700" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" input: "add_621" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_806" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" input: "^AssignVariableOp_700" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/key/bias/adam_m" } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/key/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/key/bias/adam_v" } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/key/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_994/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_994/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_994" op: "Mul" input: "Mul_994/x" input: "Mul_994/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_995/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_995" op: "Mul" input: "Mul_995/x" input: "clip_by_global_norm/clip_by_global_norm/_184" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_624" op: "AddV2" input: "Mul_994" input: "Mul_995" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_996/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_996/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_996" op: "Mul" input: "Mul_996/x" input: "Mul_996/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_184" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_184" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_997/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_997" op: "Mul" input: "Mul_997/x" input: "Square_184" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_625" op: "AddV2" input: "Mul_996" input: "Mul_997" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_184" op: "Sqrt" input: "add_625" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_626/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_626" op: "AddV2" input: "Sqrt_184" input: "add_626/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_185" op: "RealDiv" input: "add_624" input: "add_626" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_998" op: "Mul" input: "add" input: "truediv_185" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_807" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_185" op: "Sub" input: "ReadVariableOp_807" input: "mul_998" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_701" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias" input: "sub_185" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_808" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias" input: "^AssignVariableOp_701" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_702" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m" input: "add_624" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_809" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m" input: "^AssignVariableOp_702" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_703" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v" input: "add_625" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_810" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v" input: "^AssignVariableOp_703" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_999/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_999/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_999" op: "Mul" input: "Mul_999/x" input: "Mul_999/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1000/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1000" op: "Mul" input: "Mul_1000/x" input: "clip_by_global_norm/clip_by_global_norm/_185" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_627" op: "AddV2" input: "Mul_999" input: "Mul_1000" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1001/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1001/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1001" op: "Mul" input: "Mul_1001/x" input: "Mul_1001/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_185" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_185" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1002/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1002" op: "Mul" input: "Mul_1002/x" input: "Square_185" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_628" op: "AddV2" input: "Mul_1001" input: "Mul_1002" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_185" op: "Sqrt" input: "add_628" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_629/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_629" op: "AddV2" input: "Sqrt_185" input: "add_629/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_186" op: "RealDiv" input: "add_627" input: "add_629" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_811" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_1003/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_1003" op: "Mul" input: "mul_1003/x" input: "ReadVariableOp_811" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_630" op: "AddV2" input: "truediv_186" input: "mul_1003" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_1004" op: "Mul" input: "add" input: "add_630" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_812" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_186" op: "Sub" input: "ReadVariableOp_812" input: "mul_1004" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_704" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel" input: "sub_186" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_813" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel" input: "^AssignVariableOp_704" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_705" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" input: "add_627" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_814" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" input: "^AssignVariableOp_705" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_706" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" input: "add_628" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_815" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" input: "^AssignVariableOp_706" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/value/bias/adam_m" } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/self/value/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/self/value/bias/adam_v" } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/self/value/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1005/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1005/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1005" op: "Mul" input: "Mul_1005/x" input: "Mul_1005/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1006/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1006" op: "Mul" input: "Mul_1006/x" input: "clip_by_global_norm/clip_by_global_norm/_186" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_631" op: "AddV2" input: "Mul_1005" input: "Mul_1006" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1007/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1007/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1007" op: "Mul" input: "Mul_1007/x" input: "Mul_1007/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_186" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_186" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1008/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1008" op: "Mul" input: "Mul_1008/x" input: "Square_186" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_632" op: "AddV2" input: "Mul_1007" input: "Mul_1008" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_186" op: "Sqrt" input: "add_632" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_633/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_633" op: "AddV2" input: "Sqrt_186" input: "add_633/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_187" op: "RealDiv" input: "add_631" input: "add_633" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1009" op: "Mul" input: "add" input: "truediv_187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_816" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_187" op: "Sub" input: "ReadVariableOp_816" input: "mul_1009" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_707" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias" input: "sub_187" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_817" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias" input: "^AssignVariableOp_707" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_708" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m" input: "add_631" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_818" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m" input: "^AssignVariableOp_708" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_709" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v" input: "add_632" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_819" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v" input: "^AssignVariableOp_709" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1010/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1010/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1010" op: "Mul" input: "Mul_1010/x" input: "Mul_1010/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1011/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1011" op: "Mul" input: "Mul_1011/x" input: "clip_by_global_norm/clip_by_global_norm/_187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_634" op: "AddV2" input: "Mul_1010" input: "Mul_1011" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1012/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1012/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1012" op: "Mul" input: "Mul_1012/x" input: "Mul_1012/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_187" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1013/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1013" op: "Mul" input: "Mul_1013/x" input: "Square_187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_635" op: "AddV2" input: "Mul_1012" input: "Mul_1013" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_187" op: "Sqrt" input: "add_635" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_636/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_636" op: "AddV2" input: "Sqrt_187" input: "add_636/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_188" op: "RealDiv" input: "add_634" input: "add_636" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_820" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_1014/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_1014" op: "Mul" input: "mul_1014/x" input: "ReadVariableOp_820" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_637" op: "AddV2" input: "truediv_188" input: "mul_1014" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_1015" op: "Mul" input: "add" input: "add_637" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_821" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_188" op: "Sub" input: "ReadVariableOp_821" input: "mul_1015" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_710" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel" input: "sub_188" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_822" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel" input: "^AssignVariableOp_710" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_711" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" input: "add_634" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_823" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" input: "^AssignVariableOp_711" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_712" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" input: "add_635" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_824" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" input: "^AssignVariableOp_712" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1016/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1016/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1016" op: "Mul" input: "Mul_1016/x" input: "Mul_1016/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1017/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1017" op: "Mul" input: "Mul_1017/x" input: "clip_by_global_norm/clip_by_global_norm/_188" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_638" op: "AddV2" input: "Mul_1016" input: "Mul_1017" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1018/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1018/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1018" op: "Mul" input: "Mul_1018/x" input: "Mul_1018/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_188" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_188" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1019/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1019" op: "Mul" input: "Mul_1019/x" input: "Square_188" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_639" op: "AddV2" input: "Mul_1018" input: "Mul_1019" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_188" op: "Sqrt" input: "add_639" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_640/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_640" op: "AddV2" input: "Sqrt_188" input: "add_640/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_189" op: "RealDiv" input: "add_638" input: "add_640" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1020" op: "Mul" input: "add" input: "truediv_189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_825" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_189" op: "Sub" input: "ReadVariableOp_825" input: "mul_1020" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_713" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias" input: "sub_189" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_826" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias" input: "^AssignVariableOp_713" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_714" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" input: "add_638" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_827" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" input: "^AssignVariableOp_714" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_715" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" input: "add_639" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_828" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" input: "^AssignVariableOp_715" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1021/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1021/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1021" op: "Mul" input: "Mul_1021/x" input: "Mul_1021/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1022/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1022" op: "Mul" input: "Mul_1022/x" input: "clip_by_global_norm/clip_by_global_norm/_189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_641" op: "AddV2" input: "Mul_1021" input: "Mul_1022" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1023/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1023/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1023" op: "Mul" input: "Mul_1023/x" input: "Mul_1023/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_189" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1024/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1024" op: "Mul" input: "Mul_1024/x" input: "Square_189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_642" op: "AddV2" input: "Mul_1023" input: "Mul_1024" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_189" op: "Sqrt" input: "add_642" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_643/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_643" op: "AddV2" input: "Sqrt_189" input: "add_643/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_190" op: "RealDiv" input: "add_641" input: "add_643" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1025" op: "Mul" input: "add" input: "truediv_190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_829" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_190" op: "Sub" input: "ReadVariableOp_829" input: "mul_1025" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_716" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" input: "sub_190" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_830" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" input: "^AssignVariableOp_716" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_717" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" input: "add_641" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_831" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" input: "^AssignVariableOp_717" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_718" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" input: "add_642" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_832" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" input: "^AssignVariableOp_718" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1026/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1026/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1026" op: "Mul" input: "Mul_1026/x" input: "Mul_1026/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1027/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1027" op: "Mul" input: "Mul_1027/x" input: "clip_by_global_norm/clip_by_global_norm/_190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_644" op: "AddV2" input: "Mul_1026" input: "Mul_1027" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1028/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1028/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1028" op: "Mul" input: "Mul_1028/x" input: "Mul_1028/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_190" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1029/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1029" op: "Mul" input: "Mul_1029/x" input: "Square_190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_645" op: "AddV2" input: "Mul_1028" input: "Mul_1029" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_190" op: "Sqrt" input: "add_645" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_646/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_646" op: "AddV2" input: "Sqrt_190" input: "add_646/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_191" op: "RealDiv" input: "add_644" input: "add_646" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1030" op: "Mul" input: "add" input: "truediv_191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_833" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_191" op: "Sub" input: "ReadVariableOp_833" input: "mul_1030" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_719" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta" input: "sub_191" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_834" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta" input: "^AssignVariableOp_719" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_720" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" input: "add_644" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_835" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" input: "^AssignVariableOp_720" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_721" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" input: "add_645" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_836" op: "ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" input: "^AssignVariableOp_721" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\014\000\000" } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1031/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1031/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1031" op: "Mul" input: "Mul_1031/x" input: "Mul_1031/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_1032/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1032" op: "Mul" input: "Mul_1032/x" input: "clip_by_global_norm/clip_by_global_norm/_191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_647" op: "AddV2" input: "Mul_1031" input: "Mul_1032" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_1033/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1033/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1033" op: "Mul" input: "Mul_1033/x" input: "Mul_1033/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Square_191" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Mul_1034/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1034" op: "Mul" input: "Mul_1034/x" input: "Square_191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_648" op: "AddV2" input: "Mul_1033" input: "Mul_1034" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "Sqrt_191" op: "Sqrt" input: "add_648" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_649/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_649" op: "AddV2" input: "Sqrt_191" input: "add_649/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "truediv_192" op: "RealDiv" input: "add_647" input: "add_649" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_837" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_1035/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_1035" op: "Mul" input: "mul_1035/x" input: "ReadVariableOp_837" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "add_650" op: "AddV2" input: "truediv_192" input: "mul_1035" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "mul_1036" op: "Mul" input: "add" input: "add_650" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "ReadVariableOp_838" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_192" op: "Sub" input: "ReadVariableOp_838" input: "mul_1036" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } } node { name: "AssignVariableOp_722" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel" input: "sub_192" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_839" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel" input: "^AssignVariableOp_722" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_723" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" input: "add_647" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_840" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" input: "^AssignVariableOp_723" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_724" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" input: "add_648" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_841" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" input: "^AssignVariableOp_724" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 3072 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/intermediate/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1037/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1037/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1037" op: "Mul" input: "Mul_1037/x" input: "Mul_1037/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_1038/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1038" op: "Mul" input: "Mul_1038/x" input: "clip_by_global_norm/clip_by_global_norm/_192" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_651" op: "AddV2" input: "Mul_1037" input: "Mul_1038" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_1039/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1039/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1039" op: "Mul" input: "Mul_1039/x" input: "Mul_1039/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Square_192" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_192" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Mul_1040/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1040" op: "Mul" input: "Mul_1040/x" input: "Square_192" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_652" op: "AddV2" input: "Mul_1039" input: "Mul_1040" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "Sqrt_192" op: "Sqrt" input: "add_652" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "add_653/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_653" op: "AddV2" input: "Sqrt_192" input: "add_653/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "truediv_193" op: "RealDiv" input: "add_651" input: "add_653" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "mul_1041" op: "Mul" input: "add" input: "truediv_193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "ReadVariableOp_842" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_193" op: "Sub" input: "ReadVariableOp_842" input: "mul_1041" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } } node { name: "AssignVariableOp_725" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias" input: "sub_193" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_843" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias" input: "^AssignVariableOp_725" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_726" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" input: "add_651" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_844" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" input: "^AssignVariableOp_726" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_727" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" input: "add_652" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_845" op: "ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" input: "^AssignVariableOp_727" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/output/dense/kernel/adam_m" } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_m" input: "bert/encoder/layer_11/output/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\014\000\000\000\003\000\000" } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3072 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/output/dense/kernel/adam_v" } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_v" input: "bert/encoder/layer_11/output/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/output/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1042/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1042/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1042" op: "Mul" input: "Mul_1042/x" input: "Mul_1042/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_1043/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1043" op: "Mul" input: "Mul_1043/x" input: "clip_by_global_norm/clip_by_global_norm/_193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_654" op: "AddV2" input: "Mul_1042" input: "Mul_1043" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_1044/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1044/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1044" op: "Mul" input: "Mul_1044/x" input: "Mul_1044/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Square_193" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Mul_1045/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1045" op: "Mul" input: "Mul_1045/x" input: "Square_193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_655" op: "AddV2" input: "Mul_1044" input: "Mul_1045" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "Sqrt_193" op: "Sqrt" input: "add_655" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_656/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_656" op: "AddV2" input: "Sqrt_193" input: "add_656/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "truediv_194" op: "RealDiv" input: "add_654" input: "add_656" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_846" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_1046/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_1046" op: "Mul" input: "mul_1046/x" input: "ReadVariableOp_846" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "add_657" op: "AddV2" input: "truediv_194" input: "mul_1046" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "mul_1047" op: "Mul" input: "add" input: "add_657" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_847" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_194" op: "Sub" input: "ReadVariableOp_847" input: "mul_1047" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_728" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/kernel" input: "sub_194" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_848" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/kernel" input: "^AssignVariableOp_728" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_729" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_m" input: "add_654" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_849" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_m" input: "^AssignVariableOp_729" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_730" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_v" input: "add_655" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_850" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_v" input: "^AssignVariableOp_730" attr { key: "_output_shapes" value { list { shape { dim { size: 3072 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/output/dense/bias/adam_m" } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/bias/adam_m" input: "bert/encoder/layer_11/output/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/output/dense/bias/adam_v" } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/bias/adam_v" input: "bert/encoder/layer_11/output/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/output/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1048/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1048/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1048" op: "Mul" input: "Mul_1048/x" input: "Mul_1048/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1049/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1049" op: "Mul" input: "Mul_1049/x" input: "clip_by_global_norm/clip_by_global_norm/_194" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_658" op: "AddV2" input: "Mul_1048" input: "Mul_1049" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1050/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1050/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1050" op: "Mul" input: "Mul_1050/x" input: "Mul_1050/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_194" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_194" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1051/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1051" op: "Mul" input: "Mul_1051/x" input: "Square_194" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_659" op: "AddV2" input: "Mul_1050" input: "Mul_1051" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_194" op: "Sqrt" input: "add_659" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_660/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_660" op: "AddV2" input: "Sqrt_194" input: "add_660/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_195" op: "RealDiv" input: "add_658" input: "add_660" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1052" op: "Mul" input: "add" input: "truediv_195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_851" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_195" op: "Sub" input: "ReadVariableOp_851" input: "mul_1052" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_731" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/bias" input: "sub_195" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_852" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/bias" input: "^AssignVariableOp_731" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_732" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/bias/adam_m" input: "add_658" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_853" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/bias/adam_m" input: "^AssignVariableOp_732" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_733" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/bias/adam_v" input: "add_659" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_854" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/dense/bias/adam_v" input: "^AssignVariableOp_733" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1053/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1053/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1053" op: "Mul" input: "Mul_1053/x" input: "Mul_1053/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1054/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1054" op: "Mul" input: "Mul_1054/x" input: "clip_by_global_norm/clip_by_global_norm/_195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_661" op: "AddV2" input: "Mul_1053" input: "Mul_1054" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1055/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1055/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1055" op: "Mul" input: "Mul_1055/x" input: "Mul_1055/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_195" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1056/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1056" op: "Mul" input: "Mul_1056/x" input: "Square_195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_662" op: "AddV2" input: "Mul_1055" input: "Mul_1056" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_195" op: "Sqrt" input: "add_662" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_663/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_663" op: "AddV2" input: "Sqrt_195" input: "add_663/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_196" op: "RealDiv" input: "add_661" input: "add_663" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1057" op: "Mul" input: "add" input: "truediv_196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_855" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_196" op: "Sub" input: "ReadVariableOp_855" input: "mul_1057" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_734" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma" input: "sub_196" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_856" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma" input: "^AssignVariableOp_734" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_735" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" input: "add_661" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_857" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" input: "^AssignVariableOp_735" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_736" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" input: "add_662" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_858" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" input: "^AssignVariableOp_736" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v/Assign" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1058/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1058/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1058" op: "Mul" input: "Mul_1058/x" input: "Mul_1058/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1059/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1059" op: "Mul" input: "Mul_1059/x" input: "clip_by_global_norm/clip_by_global_norm/_196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_664" op: "AddV2" input: "Mul_1058" input: "Mul_1059" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1060/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1060/ReadVariableOp" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1060" op: "Mul" input: "Mul_1060/x" input: "Mul_1060/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_196" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1061/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1061" op: "Mul" input: "Mul_1061/x" input: "Square_196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_665" op: "AddV2" input: "Mul_1060" input: "Mul_1061" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_196" op: "Sqrt" input: "add_665" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_666/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_666" op: "AddV2" input: "Sqrt_196" input: "add_666/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_197" op: "RealDiv" input: "add_664" input: "add_666" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1062" op: "Mul" input: "add" input: "truediv_197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_859" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_197" op: "Sub" input: "ReadVariableOp_859" input: "mul_1062" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_737" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta" input: "sub_197" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_860" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta" input: "^AssignVariableOp_737" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_738" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" input: "add_664" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_861" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" input: "^AssignVariableOp_738" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_739" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" input: "add_665" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_862" op: "ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" input: "^AssignVariableOp_739" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/pooler/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/pooler/dense/kernel/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/pooler/dense/kernel/adam_m/Initializer/zeros" op: "Fill" input: "bert/pooler/dense/kernel/adam_m/Initializer/zeros/shape_as_tensor" input: "bert/pooler/dense/kernel/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/pooler/dense/kernel/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/pooler/dense/kernel/adam_m" } } } node { name: "bert/pooler/dense/kernel/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/pooler/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/pooler/dense/kernel/adam_m/Assign" op: "AssignVariableOp" input: "bert/pooler/dense/kernel/adam_m" input: "bert/pooler/dense/kernel/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/pooler/dense/kernel/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/pooler/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/pooler/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\000\003\000\000\000\003\000\000" } } } } node { name: "bert/pooler/dense/kernel/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "bert/pooler/dense/kernel/adam_v/Initializer/zeros" op: "Fill" input: "bert/pooler/dense/kernel/adam_v/Initializer/zeros/shape_as_tensor" input: "bert/pooler/dense/kernel/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "bert/pooler/dense/kernel/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/kernel/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/pooler/dense/kernel/adam_v" } } } node { name: "bert/pooler/dense/kernel/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/pooler/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/pooler/dense/kernel/adam_v/Assign" op: "AssignVariableOp" input: "bert/pooler/dense/kernel/adam_v" input: "bert/pooler/dense/kernel/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/pooler/dense/kernel/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/pooler/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1063/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1063/ReadVariableOp" op: "ReadVariableOp" input: "bert/pooler/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1063" op: "Mul" input: "Mul_1063/x" input: "Mul_1063/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1064/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1064" op: "Mul" input: "Mul_1064/x" input: "clip_by_global_norm/clip_by_global_norm/_197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_667" op: "AddV2" input: "Mul_1063" input: "Mul_1064" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1065/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1065/ReadVariableOp" op: "ReadVariableOp" input: "bert/pooler/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1065" op: "Mul" input: "Mul_1065/x" input: "Mul_1065/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Square_197" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Mul_1066/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1066" op: "Mul" input: "Mul_1066/x" input: "Square_197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_668" op: "AddV2" input: "Mul_1065" input: "Mul_1066" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "Sqrt_197" op: "Sqrt" input: "add_668" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_669/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_669" op: "AddV2" input: "Sqrt_197" input: "add_669/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "truediv_198" op: "RealDiv" input: "add_667" input: "add_669" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_863" op: "ReadVariableOp" input: "bert/pooler/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_1067/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_1067" op: "Mul" input: "mul_1067/x" input: "ReadVariableOp_863" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "add_670" op: "AddV2" input: "truediv_198" input: "mul_1067" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "mul_1068" op: "Mul" input: "add" input: "add_670" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_864" op: "ReadVariableOp" input: "bert/pooler/dense/kernel" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_198" op: "Sub" input: "ReadVariableOp_864" input: "mul_1068" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_740" op: "AssignVariableOp" input: "bert/pooler/dense/kernel" input: "sub_198" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_865" op: "ReadVariableOp" input: "bert/pooler/dense/kernel" input: "^AssignVariableOp_740" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_741" op: "AssignVariableOp" input: "bert/pooler/dense/kernel/adam_m" input: "add_667" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_866" op: "ReadVariableOp" input: "bert/pooler/dense/kernel/adam_m" input: "^AssignVariableOp_741" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_742" op: "AssignVariableOp" input: "bert/pooler/dense/kernel/adam_v" input: "add_668" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_867" op: "ReadVariableOp" input: "bert/pooler/dense/kernel/adam_v" input: "^AssignVariableOp_742" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/pooler/dense/bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/pooler/dense/bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/pooler/dense/bias/adam_m" } } } node { name: "bert/pooler/dense/bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/pooler/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/pooler/dense/bias/adam_m/Assign" op: "AssignVariableOp" input: "bert/pooler/dense/bias/adam_m" input: "bert/pooler/dense/bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/pooler/dense/bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/pooler/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "bert/pooler/dense/bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 768 } } float_val: 0.0 } } } } node { name: "bert/pooler/dense/bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@bert/pooler/dense/bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 768 } } } } attr { key: "shared_name" value { s: "bert/pooler/dense/bias/adam_v" } } } node { name: "bert/pooler/dense/bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "bert/pooler/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "bert/pooler/dense/bias/adam_v/Assign" op: "AssignVariableOp" input: "bert/pooler/dense/bias/adam_v" input: "bert/pooler/dense/bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "bert/pooler/dense/bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "bert/pooler/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1069/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1069/ReadVariableOp" op: "ReadVariableOp" input: "bert/pooler/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1069" op: "Mul" input: "Mul_1069/x" input: "Mul_1069/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1070/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1070" op: "Mul" input: "Mul_1070/x" input: "clip_by_global_norm/clip_by_global_norm/_198" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_671" op: "AddV2" input: "Mul_1069" input: "Mul_1070" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1071/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1071/ReadVariableOp" op: "ReadVariableOp" input: "bert/pooler/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1071" op: "Mul" input: "Mul_1071/x" input: "Mul_1071/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Square_198" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_198" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Mul_1072/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1072" op: "Mul" input: "Mul_1072/x" input: "Square_198" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_672" op: "AddV2" input: "Mul_1071" input: "Mul_1072" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "Sqrt_198" op: "Sqrt" input: "add_672" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "add_673/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_673" op: "AddV2" input: "Sqrt_198" input: "add_673/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "truediv_199" op: "RealDiv" input: "add_671" input: "add_673" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "mul_1073" op: "Mul" input: "add" input: "truediv_199" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "ReadVariableOp_868" op: "ReadVariableOp" input: "bert/pooler/dense/bias" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_199" op: "Sub" input: "ReadVariableOp_868" input: "mul_1073" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } } node { name: "AssignVariableOp_743" op: "AssignVariableOp" input: "bert/pooler/dense/bias" input: "sub_199" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_869" op: "ReadVariableOp" input: "bert/pooler/dense/bias" input: "^AssignVariableOp_743" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_744" op: "AssignVariableOp" input: "bert/pooler/dense/bias/adam_m" input: "add_671" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_870" op: "ReadVariableOp" input: "bert/pooler/dense/bias/adam_m" input: "^AssignVariableOp_744" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_745" op: "AssignVariableOp" input: "bert/pooler/dense/bias/adam_v" input: "add_672" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_871" op: "ReadVariableOp" input: "bert/pooler/dense/bias/adam_v" input: "^AssignVariableOp_745" attr { key: "_output_shapes" value { list { shape { dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "output_weights/adam_m/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@output_weights/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\003\000\000\000\000\003\000\000" } } } } node { name: "output_weights/adam_m/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@output_weights/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "output_weights/adam_m/Initializer/zeros" op: "Fill" input: "output_weights/adam_m/Initializer/zeros/shape_as_tensor" input: "output_weights/adam_m/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@output_weights/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "output_weights/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@output_weights/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "output_weights/adam_m" } } } node { name: "output_weights/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "output_weights/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "output_weights/adam_m/Assign" op: "AssignVariableOp" input: "output_weights/adam_m" input: "output_weights/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "output_weights/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "output_weights/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "output_weights/adam_v/Initializer/zeros/shape_as_tensor" op: "Const" attr { key: "_class" value { list { s: "loc:@output_weights/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 2 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 2 } } tensor_content: "\003\000\000\000\000\003\000\000" } } } } node { name: "output_weights/adam_v/Initializer/zeros/Const" op: "Const" attr { key: "_class" value { list { s: "loc:@output_weights/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0 } } } } node { name: "output_weights/adam_v/Initializer/zeros" op: "Fill" input: "output_weights/adam_v/Initializer/zeros/shape_as_tensor" input: "output_weights/adam_v/Initializer/zeros/Const" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_class" value { list { s: "loc:@output_weights/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } attr { key: "index_type" value { type: DT_INT32 } } } node { name: "output_weights/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@output_weights/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3 } dim { size: 768 } } } } attr { key: "shared_name" value { s: "output_weights/adam_v" } } } node { name: "output_weights/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "output_weights/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "output_weights/adam_v/Assign" op: "AssignVariableOp" input: "output_weights/adam_v" input: "output_weights/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "output_weights/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "output_weights/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1074/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1074/ReadVariableOp" op: "ReadVariableOp" input: "output_weights/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1074" op: "Mul" input: "Mul_1074/x" input: "Mul_1074/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "Mul_1075/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1075" op: "Mul" input: "Mul_1075/x" input: "clip_by_global_norm/clip_by_global_norm/_199" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "add_674" op: "AddV2" input: "Mul_1074" input: "Mul_1075" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "Mul_1076/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1076/ReadVariableOp" op: "ReadVariableOp" input: "output_weights/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1076" op: "Mul" input: "Mul_1076/x" input: "Mul_1076/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "Square_199" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_199" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "Mul_1077/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1077" op: "Mul" input: "Mul_1077/x" input: "Square_199" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "add_675" op: "AddV2" input: "Mul_1076" input: "Mul_1077" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "Sqrt_199" op: "Sqrt" input: "add_675" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "add_676/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_676" op: "AddV2" input: "Sqrt_199" input: "add_676/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "truediv_200" op: "RealDiv" input: "add_674" input: "add_676" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_872" op: "ReadVariableOp" input: "output_weights" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "mul_1078/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.009999999776482582 } } } } node { name: "mul_1078" op: "Mul" input: "mul_1078/x" input: "ReadVariableOp_872" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "add_677" op: "AddV2" input: "truediv_200" input: "mul_1078" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "mul_1079" op: "Mul" input: "add" input: "add_677" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "ReadVariableOp_873" op: "ReadVariableOp" input: "output_weights" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_200" op: "Sub" input: "ReadVariableOp_873" input: "mul_1079" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } } node { name: "AssignVariableOp_746" op: "AssignVariableOp" input: "output_weights" input: "sub_200" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_874" op: "ReadVariableOp" input: "output_weights" input: "^AssignVariableOp_746" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_747" op: "AssignVariableOp" input: "output_weights/adam_m" input: "add_674" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_875" op: "ReadVariableOp" input: "output_weights/adam_m" input: "^AssignVariableOp_747" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_748" op: "AssignVariableOp" input: "output_weights/adam_v" input: "add_675" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_876" op: "ReadVariableOp" input: "output_weights/adam_v" input: "^AssignVariableOp_748" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } dim { size: 768 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "output_bias/adam_m/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 3 } } float_val: 0.0 } } } } node { name: "output_bias/adam_m" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@output_bias/adam_m" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3 } } } } attr { key: "shared_name" value { s: "output_bias/adam_m" } } } node { name: "output_bias/adam_m/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "output_bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "output_bias/adam_m/Assign" op: "AssignVariableOp" input: "output_bias/adam_m" input: "output_bias/adam_m/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "output_bias/adam_m/Read/ReadVariableOp" op: "ReadVariableOp" input: "output_bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "output_bias/adam_v/Initializer/zeros" op: "Const" attr { key: "_class" value { list { s: "loc:@output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { dim { size: 3 } } float_val: 0.0 } } } } node { name: "output_bias/adam_v" op: "VarHandleOp" attr { key: "_class" value { list { s: "loc:@output_bias/adam_v" } } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "allowed_devices" value { list { } } } attr { key: "container" value { s: "" } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "shape" value { shape { dim { size: 3 } } } } attr { key: "shared_name" value { s: "output_bias/adam_v" } } } node { name: "output_bias/adam_v/IsInitialized/VarIsInitializedOp" op: "VarIsInitializedOp" input: "output_bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "output_bias/adam_v/Assign" op: "AssignVariableOp" input: "output_bias/adam_v" input: "output_bias/adam_v/Initializer/zeros" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "output_bias/adam_v/Read/ReadVariableOp" op: "ReadVariableOp" input: "output_bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1080/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.8999999761581421 } } } } node { name: "Mul_1080/ReadVariableOp" op: "ReadVariableOp" input: "output_bias/adam_m" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1080" op: "Mul" input: "Mul_1080/x" input: "Mul_1080/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "Mul_1081/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.10000000149011612 } } } } node { name: "Mul_1081" op: "Mul" input: "Mul_1081/x" input: "clip_by_global_norm/clip_by_global_norm/_200" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "add_678" op: "AddV2" input: "Mul_1080" input: "Mul_1081" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "Mul_1082/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.9990000128746033 } } } } node { name: "Mul_1082/ReadVariableOp" op: "ReadVariableOp" input: "output_bias/adam_v" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "Mul_1082" op: "Mul" input: "Mul_1082/x" input: "Mul_1082/ReadVariableOp" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "Square_200" op: "Square" input: "clip_by_global_norm/clip_by_global_norm/_200" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "Mul_1083/x" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 0.0010000000474974513 } } } } node { name: "Mul_1083" op: "Mul" input: "Mul_1083/x" input: "Square_200" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "add_679" op: "AddV2" input: "Mul_1082" input: "Mul_1083" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "Sqrt_200" op: "Sqrt" input: "add_679" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "add_680/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "value" value { tensor { dtype: DT_FLOAT tensor_shape { } float_val: 9.999999974752427e-07 } } } } node { name: "add_680" op: "AddV2" input: "Sqrt_200" input: "add_680/y" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "truediv_201" op: "RealDiv" input: "add_678" input: "add_680" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "mul_1084" op: "Mul" input: "add" input: "truediv_201" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "ReadVariableOp_877" op: "ReadVariableOp" input: "output_bias" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "sub_201" op: "Sub" input: "ReadVariableOp_877" input: "mul_1084" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } } node { name: "AssignVariableOp_749" op: "AssignVariableOp" input: "output_bias" input: "sub_201" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_878" op: "ReadVariableOp" input: "output_bias" input: "^AssignVariableOp_749" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_750" op: "AssignVariableOp" input: "output_bias/adam_m" input: "add_678" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_879" op: "ReadVariableOp" input: "output_bias/adam_m" input: "^AssignVariableOp_750" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "AssignVariableOp_751" op: "AssignVariableOp" input: "output_bias/adam_v" input: "add_679" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_880" op: "ReadVariableOp" input: "output_bias/adam_v" input: "^AssignVariableOp_751" attr { key: "_output_shapes" value { list { shape { dim { size: 3 } } } } } attr { key: "dtype" value { type: DT_FLOAT } } } node { name: "group_deps" op: "NoOp" input: "^AssignVariableOp_149" input: "^AssignVariableOp_150" input: "^AssignVariableOp_151" input: "^AssignVariableOp_152" input: "^AssignVariableOp_153" input: "^AssignVariableOp_154" input: "^AssignVariableOp_155" input: "^AssignVariableOp_156" input: "^AssignVariableOp_157" input: "^AssignVariableOp_158" input: "^AssignVariableOp_159" input: "^AssignVariableOp_160" input: "^AssignVariableOp_161" input: "^AssignVariableOp_162" input: "^AssignVariableOp_163" input: "^AssignVariableOp_164" input: "^AssignVariableOp_165" input: "^AssignVariableOp_166" input: "^AssignVariableOp_167" input: "^AssignVariableOp_168" input: "^AssignVariableOp_169" input: "^AssignVariableOp_170" input: "^AssignVariableOp_171" input: "^AssignVariableOp_172" input: "^AssignVariableOp_173" input: "^AssignVariableOp_174" input: "^AssignVariableOp_175" input: "^AssignVariableOp_176" input: "^AssignVariableOp_177" input: "^AssignVariableOp_178" input: "^AssignVariableOp_179" input: "^AssignVariableOp_180" input: "^AssignVariableOp_181" input: "^AssignVariableOp_182" input: "^AssignVariableOp_183" input: "^AssignVariableOp_184" input: "^AssignVariableOp_185" input: "^AssignVariableOp_186" input: "^AssignVariableOp_187" input: "^AssignVariableOp_188" input: "^AssignVariableOp_189" input: "^AssignVariableOp_190" input: "^AssignVariableOp_191" input: "^AssignVariableOp_192" input: "^AssignVariableOp_193" input: "^AssignVariableOp_194" input: "^AssignVariableOp_195" input: "^AssignVariableOp_196" input: "^AssignVariableOp_197" input: "^AssignVariableOp_198" input: "^AssignVariableOp_199" input: "^AssignVariableOp_200" input: "^AssignVariableOp_201" input: "^AssignVariableOp_202" input: "^AssignVariableOp_203" input: "^AssignVariableOp_204" input: "^AssignVariableOp_205" input: "^AssignVariableOp_206" input: "^AssignVariableOp_207" input: "^AssignVariableOp_208" input: "^AssignVariableOp_209" input: "^AssignVariableOp_210" input: "^AssignVariableOp_211" input: "^AssignVariableOp_212" input: "^AssignVariableOp_213" input: "^AssignVariableOp_214" input: "^AssignVariableOp_215" input: "^AssignVariableOp_216" input: "^AssignVariableOp_217" input: "^AssignVariableOp_218" input: "^AssignVariableOp_219" input: "^AssignVariableOp_220" input: "^AssignVariableOp_221" input: "^AssignVariableOp_222" input: "^AssignVariableOp_223" input: "^AssignVariableOp_224" input: "^AssignVariableOp_225" input: "^AssignVariableOp_226" input: "^AssignVariableOp_227" input: "^AssignVariableOp_228" input: "^AssignVariableOp_229" input: "^AssignVariableOp_230" input: "^AssignVariableOp_231" input: "^AssignVariableOp_232" input: "^AssignVariableOp_233" input: "^AssignVariableOp_234" input: "^AssignVariableOp_235" input: "^AssignVariableOp_236" input: "^AssignVariableOp_237" input: "^AssignVariableOp_238" input: "^AssignVariableOp_239" input: "^AssignVariableOp_240" input: "^AssignVariableOp_241" input: "^AssignVariableOp_242" input: "^AssignVariableOp_243" input: "^AssignVariableOp_244" input: "^AssignVariableOp_245" input: "^AssignVariableOp_246" input: "^AssignVariableOp_247" input: "^AssignVariableOp_248" input: "^AssignVariableOp_249" input: "^AssignVariableOp_250" input: "^AssignVariableOp_251" input: "^AssignVariableOp_252" input: "^AssignVariableOp_253" input: "^AssignVariableOp_254" input: "^AssignVariableOp_255" input: "^AssignVariableOp_256" input: "^AssignVariableOp_257" input: "^AssignVariableOp_258" input: "^AssignVariableOp_259" input: "^AssignVariableOp_260" input: "^AssignVariableOp_261" input: "^AssignVariableOp_262" input: "^AssignVariableOp_263" input: "^AssignVariableOp_264" input: "^AssignVariableOp_265" input: "^AssignVariableOp_266" input: "^AssignVariableOp_267" input: "^AssignVariableOp_268" input: "^AssignVariableOp_269" input: "^AssignVariableOp_270" input: "^AssignVariableOp_271" input: "^AssignVariableOp_272" input: "^AssignVariableOp_273" input: "^AssignVariableOp_274" input: "^AssignVariableOp_275" input: "^AssignVariableOp_276" input: "^AssignVariableOp_277" input: "^AssignVariableOp_278" input: "^AssignVariableOp_279" input: "^AssignVariableOp_280" input: "^AssignVariableOp_281" input: "^AssignVariableOp_282" input: "^AssignVariableOp_283" input: "^AssignVariableOp_284" input: "^AssignVariableOp_285" input: "^AssignVariableOp_286" input: "^AssignVariableOp_287" input: "^AssignVariableOp_288" input: "^AssignVariableOp_289" input: "^AssignVariableOp_290" input: "^AssignVariableOp_291" input: "^AssignVariableOp_292" input: "^AssignVariableOp_293" input: "^AssignVariableOp_294" input: "^AssignVariableOp_295" input: "^AssignVariableOp_296" input: "^AssignVariableOp_297" input: "^AssignVariableOp_298" input: "^AssignVariableOp_299" input: "^AssignVariableOp_300" input: "^AssignVariableOp_301" input: "^AssignVariableOp_302" input: "^AssignVariableOp_303" input: "^AssignVariableOp_304" input: "^AssignVariableOp_305" input: "^AssignVariableOp_306" input: "^AssignVariableOp_307" input: "^AssignVariableOp_308" input: "^AssignVariableOp_309" input: "^AssignVariableOp_310" input: "^AssignVariableOp_311" input: "^AssignVariableOp_312" input: "^AssignVariableOp_313" input: "^AssignVariableOp_314" input: "^AssignVariableOp_315" input: "^AssignVariableOp_316" input: "^AssignVariableOp_317" input: "^AssignVariableOp_318" input: "^AssignVariableOp_319" input: "^AssignVariableOp_320" input: "^AssignVariableOp_321" input: "^AssignVariableOp_322" input: "^AssignVariableOp_323" input: "^AssignVariableOp_324" input: "^AssignVariableOp_325" input: "^AssignVariableOp_326" input: "^AssignVariableOp_327" input: "^AssignVariableOp_328" input: "^AssignVariableOp_329" input: "^AssignVariableOp_330" input: "^AssignVariableOp_331" input: "^AssignVariableOp_332" input: "^AssignVariableOp_333" input: "^AssignVariableOp_334" input: "^AssignVariableOp_335" input: "^AssignVariableOp_336" input: "^AssignVariableOp_337" input: "^AssignVariableOp_338" input: "^AssignVariableOp_339" input: "^AssignVariableOp_340" input: "^AssignVariableOp_341" input: "^AssignVariableOp_342" input: "^AssignVariableOp_343" input: "^AssignVariableOp_344" input: "^AssignVariableOp_345" input: "^AssignVariableOp_346" input: "^AssignVariableOp_347" input: "^AssignVariableOp_348" input: "^AssignVariableOp_349" input: "^AssignVariableOp_350" input: "^AssignVariableOp_351" input: "^AssignVariableOp_352" input: "^AssignVariableOp_353" input: "^AssignVariableOp_354" input: "^AssignVariableOp_355" input: "^AssignVariableOp_356" input: "^AssignVariableOp_357" input: "^AssignVariableOp_358" input: "^AssignVariableOp_359" input: "^AssignVariableOp_360" input: "^AssignVariableOp_361" input: "^AssignVariableOp_362" input: "^AssignVariableOp_363" input: "^AssignVariableOp_364" input: "^AssignVariableOp_365" input: "^AssignVariableOp_366" input: "^AssignVariableOp_367" input: "^AssignVariableOp_368" input: "^AssignVariableOp_369" input: "^AssignVariableOp_370" input: "^AssignVariableOp_371" input: "^AssignVariableOp_372" input: "^AssignVariableOp_373" input: "^AssignVariableOp_374" input: "^AssignVariableOp_375" input: "^AssignVariableOp_376" input: "^AssignVariableOp_377" input: "^AssignVariableOp_378" input: "^AssignVariableOp_379" input: "^AssignVariableOp_380" input: "^AssignVariableOp_381" input: "^AssignVariableOp_382" input: "^AssignVariableOp_383" input: "^AssignVariableOp_384" input: "^AssignVariableOp_385" input: "^AssignVariableOp_386" input: "^AssignVariableOp_387" input: "^AssignVariableOp_388" input: "^AssignVariableOp_389" input: "^AssignVariableOp_390" input: "^AssignVariableOp_391" input: "^AssignVariableOp_392" input: "^AssignVariableOp_393" input: "^AssignVariableOp_394" input: "^AssignVariableOp_395" input: "^AssignVariableOp_396" input: "^AssignVariableOp_397" input: "^AssignVariableOp_398" input: "^AssignVariableOp_399" input: "^AssignVariableOp_400" input: "^AssignVariableOp_401" input: "^AssignVariableOp_402" input: "^AssignVariableOp_403" input: "^AssignVariableOp_404" input: "^AssignVariableOp_405" input: "^AssignVariableOp_406" input: "^AssignVariableOp_407" input: "^AssignVariableOp_408" input: "^AssignVariableOp_409" input: "^AssignVariableOp_410" input: "^AssignVariableOp_411" input: "^AssignVariableOp_412" input: "^AssignVariableOp_413" input: "^AssignVariableOp_414" input: "^AssignVariableOp_415" input: "^AssignVariableOp_416" input: "^AssignVariableOp_417" input: "^AssignVariableOp_418" input: "^AssignVariableOp_419" input: "^AssignVariableOp_420" input: "^AssignVariableOp_421" input: "^AssignVariableOp_422" input: "^AssignVariableOp_423" input: "^AssignVariableOp_424" input: "^AssignVariableOp_425" input: "^AssignVariableOp_426" input: "^AssignVariableOp_427" input: "^AssignVariableOp_428" input: "^AssignVariableOp_429" input: "^AssignVariableOp_430" input: "^AssignVariableOp_431" input: "^AssignVariableOp_432" input: "^AssignVariableOp_433" input: "^AssignVariableOp_434" input: "^AssignVariableOp_435" input: "^AssignVariableOp_436" input: "^AssignVariableOp_437" input: "^AssignVariableOp_438" input: "^AssignVariableOp_439" input: "^AssignVariableOp_440" input: "^AssignVariableOp_441" input: "^AssignVariableOp_442" input: "^AssignVariableOp_443" input: "^AssignVariableOp_444" input: "^AssignVariableOp_445" input: "^AssignVariableOp_446" input: "^AssignVariableOp_447" input: "^AssignVariableOp_448" input: "^AssignVariableOp_449" input: "^AssignVariableOp_450" input: "^AssignVariableOp_451" input: "^AssignVariableOp_452" input: "^AssignVariableOp_453" input: "^AssignVariableOp_454" input: "^AssignVariableOp_455" input: "^AssignVariableOp_456" input: "^AssignVariableOp_457" input: "^AssignVariableOp_458" input: "^AssignVariableOp_459" input: "^AssignVariableOp_460" input: "^AssignVariableOp_461" input: "^AssignVariableOp_462" input: "^AssignVariableOp_463" input: "^AssignVariableOp_464" input: "^AssignVariableOp_465" input: "^AssignVariableOp_466" input: "^AssignVariableOp_467" input: "^AssignVariableOp_468" input: "^AssignVariableOp_469" input: "^AssignVariableOp_470" input: "^AssignVariableOp_471" input: "^AssignVariableOp_472" input: "^AssignVariableOp_473" input: "^AssignVariableOp_474" input: "^AssignVariableOp_475" input: "^AssignVariableOp_476" input: "^AssignVariableOp_477" input: "^AssignVariableOp_478" input: "^AssignVariableOp_479" input: "^AssignVariableOp_480" input: "^AssignVariableOp_481" input: "^AssignVariableOp_482" input: "^AssignVariableOp_483" input: "^AssignVariableOp_484" input: "^AssignVariableOp_485" input: "^AssignVariableOp_486" input: "^AssignVariableOp_487" input: "^AssignVariableOp_488" input: "^AssignVariableOp_489" input: "^AssignVariableOp_490" input: "^AssignVariableOp_491" input: "^AssignVariableOp_492" input: "^AssignVariableOp_493" input: "^AssignVariableOp_494" input: "^AssignVariableOp_495" input: "^AssignVariableOp_496" input: "^AssignVariableOp_497" input: "^AssignVariableOp_498" input: "^AssignVariableOp_499" input: "^AssignVariableOp_500" input: "^AssignVariableOp_501" input: "^AssignVariableOp_502" input: "^AssignVariableOp_503" input: "^AssignVariableOp_504" input: "^AssignVariableOp_505" input: "^AssignVariableOp_506" input: "^AssignVariableOp_507" input: "^AssignVariableOp_508" input: "^AssignVariableOp_509" input: "^AssignVariableOp_510" input: "^AssignVariableOp_511" input: "^AssignVariableOp_512" input: "^AssignVariableOp_513" input: "^AssignVariableOp_514" input: "^AssignVariableOp_515" input: "^AssignVariableOp_516" input: "^AssignVariableOp_517" input: "^AssignVariableOp_518" input: "^AssignVariableOp_519" input: "^AssignVariableOp_520" input: "^AssignVariableOp_521" input: "^AssignVariableOp_522" input: "^AssignVariableOp_523" input: "^AssignVariableOp_524" input: "^AssignVariableOp_525" input: "^AssignVariableOp_526" input: "^AssignVariableOp_527" input: "^AssignVariableOp_528" input: "^AssignVariableOp_529" input: "^AssignVariableOp_530" input: "^AssignVariableOp_531" input: "^AssignVariableOp_532" input: "^AssignVariableOp_533" input: "^AssignVariableOp_534" input: "^AssignVariableOp_535" input: "^AssignVariableOp_536" input: "^AssignVariableOp_537" input: "^AssignVariableOp_538" input: "^AssignVariableOp_539" input: "^AssignVariableOp_540" input: "^AssignVariableOp_541" input: "^AssignVariableOp_542" input: "^AssignVariableOp_543" input: "^AssignVariableOp_544" input: "^AssignVariableOp_545" input: "^AssignVariableOp_546" input: "^AssignVariableOp_547" input: "^AssignVariableOp_548" input: "^AssignVariableOp_549" input: "^AssignVariableOp_550" input: "^AssignVariableOp_551" input: "^AssignVariableOp_552" input: "^AssignVariableOp_553" input: "^AssignVariableOp_554" input: "^AssignVariableOp_555" input: "^AssignVariableOp_556" input: "^AssignVariableOp_557" input: "^AssignVariableOp_558" input: "^AssignVariableOp_559" input: "^AssignVariableOp_560" input: "^AssignVariableOp_561" input: "^AssignVariableOp_562" input: "^AssignVariableOp_563" input: "^AssignVariableOp_564" input: "^AssignVariableOp_565" input: "^AssignVariableOp_566" input: "^AssignVariableOp_567" input: "^AssignVariableOp_568" input: "^AssignVariableOp_569" input: "^AssignVariableOp_570" input: "^AssignVariableOp_571" input: "^AssignVariableOp_572" input: "^AssignVariableOp_573" input: "^AssignVariableOp_574" input: "^AssignVariableOp_575" input: "^AssignVariableOp_576" input: "^AssignVariableOp_577" input: "^AssignVariableOp_578" input: "^AssignVariableOp_579" input: "^AssignVariableOp_580" input: "^AssignVariableOp_581" input: "^AssignVariableOp_582" input: "^AssignVariableOp_583" input: "^AssignVariableOp_584" input: "^AssignVariableOp_585" input: "^AssignVariableOp_586" input: "^AssignVariableOp_587" input: "^AssignVariableOp_588" input: "^AssignVariableOp_589" input: "^AssignVariableOp_590" input: "^AssignVariableOp_591" input: "^AssignVariableOp_592" input: "^AssignVariableOp_593" input: "^AssignVariableOp_594" input: "^AssignVariableOp_595" input: "^AssignVariableOp_596" input: "^AssignVariableOp_597" input: "^AssignVariableOp_598" input: "^AssignVariableOp_599" input: "^AssignVariableOp_600" input: "^AssignVariableOp_601" input: "^AssignVariableOp_602" input: "^AssignVariableOp_603" input: "^AssignVariableOp_604" input: "^AssignVariableOp_605" input: "^AssignVariableOp_606" input: "^AssignVariableOp_607" input: "^AssignVariableOp_608" input: "^AssignVariableOp_609" input: "^AssignVariableOp_610" input: "^AssignVariableOp_611" input: "^AssignVariableOp_612" input: "^AssignVariableOp_613" input: "^AssignVariableOp_614" input: "^AssignVariableOp_615" input: "^AssignVariableOp_616" input: "^AssignVariableOp_617" input: "^AssignVariableOp_618" input: "^AssignVariableOp_619" input: "^AssignVariableOp_620" input: "^AssignVariableOp_621" input: "^AssignVariableOp_622" input: "^AssignVariableOp_623" input: "^AssignVariableOp_624" input: "^AssignVariableOp_625" input: "^AssignVariableOp_626" input: "^AssignVariableOp_627" input: "^AssignVariableOp_628" input: "^AssignVariableOp_629" input: "^AssignVariableOp_630" input: "^AssignVariableOp_631" input: "^AssignVariableOp_632" input: "^AssignVariableOp_633" input: "^AssignVariableOp_634" input: "^AssignVariableOp_635" input: "^AssignVariableOp_636" input: "^AssignVariableOp_637" input: "^AssignVariableOp_638" input: "^AssignVariableOp_639" input: "^AssignVariableOp_640" input: "^AssignVariableOp_641" input: "^AssignVariableOp_642" input: "^AssignVariableOp_643" input: "^AssignVariableOp_644" input: "^AssignVariableOp_645" input: "^AssignVariableOp_646" input: "^AssignVariableOp_647" input: "^AssignVariableOp_648" input: "^AssignVariableOp_649" input: "^AssignVariableOp_650" input: "^AssignVariableOp_651" input: "^AssignVariableOp_652" input: "^AssignVariableOp_653" input: "^AssignVariableOp_654" input: "^AssignVariableOp_655" input: "^AssignVariableOp_656" input: "^AssignVariableOp_657" input: "^AssignVariableOp_658" input: "^AssignVariableOp_659" input: "^AssignVariableOp_660" input: "^AssignVariableOp_661" input: "^AssignVariableOp_662" input: "^AssignVariableOp_663" input: "^AssignVariableOp_664" input: "^AssignVariableOp_665" input: "^AssignVariableOp_666" input: "^AssignVariableOp_667" input: "^AssignVariableOp_668" input: "^AssignVariableOp_669" input: "^AssignVariableOp_670" input: "^AssignVariableOp_671" input: "^AssignVariableOp_672" input: "^AssignVariableOp_673" input: "^AssignVariableOp_674" input: "^AssignVariableOp_675" input: "^AssignVariableOp_676" input: "^AssignVariableOp_677" input: "^AssignVariableOp_678" input: "^AssignVariableOp_679" input: "^AssignVariableOp_680" input: "^AssignVariableOp_681" input: "^AssignVariableOp_682" input: "^AssignVariableOp_683" input: "^AssignVariableOp_684" input: "^AssignVariableOp_685" input: "^AssignVariableOp_686" input: "^AssignVariableOp_687" input: "^AssignVariableOp_688" input: "^AssignVariableOp_689" input: "^AssignVariableOp_690" input: "^AssignVariableOp_691" input: "^AssignVariableOp_692" input: "^AssignVariableOp_693" input: "^AssignVariableOp_694" input: "^AssignVariableOp_695" input: "^AssignVariableOp_696" input: "^AssignVariableOp_697" input: "^AssignVariableOp_698" input: "^AssignVariableOp_699" input: "^AssignVariableOp_700" input: "^AssignVariableOp_701" input: "^AssignVariableOp_702" input: "^AssignVariableOp_703" input: "^AssignVariableOp_704" input: "^AssignVariableOp_705" input: "^AssignVariableOp_706" input: "^AssignVariableOp_707" input: "^AssignVariableOp_708" input: "^AssignVariableOp_709" input: "^AssignVariableOp_710" input: "^AssignVariableOp_711" input: "^AssignVariableOp_712" input: "^AssignVariableOp_713" input: "^AssignVariableOp_714" input: "^AssignVariableOp_715" input: "^AssignVariableOp_716" input: "^AssignVariableOp_717" input: "^AssignVariableOp_718" input: "^AssignVariableOp_719" input: "^AssignVariableOp_720" input: "^AssignVariableOp_721" input: "^AssignVariableOp_722" input: "^AssignVariableOp_723" input: "^AssignVariableOp_724" input: "^AssignVariableOp_725" input: "^AssignVariableOp_726" input: "^AssignVariableOp_727" input: "^AssignVariableOp_728" input: "^AssignVariableOp_729" input: "^AssignVariableOp_730" input: "^AssignVariableOp_731" input: "^AssignVariableOp_732" input: "^AssignVariableOp_733" input: "^AssignVariableOp_734" input: "^AssignVariableOp_735" input: "^AssignVariableOp_736" input: "^AssignVariableOp_737" input: "^AssignVariableOp_738" input: "^AssignVariableOp_739" input: "^AssignVariableOp_740" input: "^AssignVariableOp_741" input: "^AssignVariableOp_742" input: "^AssignVariableOp_743" input: "^AssignVariableOp_744" input: "^AssignVariableOp_745" input: "^AssignVariableOp_746" input: "^AssignVariableOp_747" input: "^AssignVariableOp_748" input: "^AssignVariableOp_749" input: "^AssignVariableOp_750" input: "^AssignVariableOp_751" attr { key: "_has_manual_control_dependencies" value { b: true } } } node { name: "ReadVariableOp_881" op: "ReadVariableOp" input: "global_step" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } } node { name: "add_681/y" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 1 } } } } node { name: "add_681" op: "AddV2" input: "ReadVariableOp_881" input: "add_681/y" attr { key: "T" value { type: DT_INT64 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "AssignVariableOp_752" op: "AssignVariableOp" input: "global_step" input: "add_681" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "validate_shape" value { b: true } } } node { name: "ReadVariableOp_882" op: "ReadVariableOp" input: "global_step" input: "^AssignVariableOp_752" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } } node { name: "group_deps_1" op: "NoOp" input: "^AssignVariableOp_752" input: "^group_deps" } node { name: "loss_1/tags" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "loss_1" } } } } node { name: "loss_1" op: "ScalarSummary" input: "loss_1/tags" input: "loss/Mean" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "init" op: "NoOp" input: "^AssignVariableOp" input: "^AssignVariableOp_1" input: "^AssignVariableOp_10" input: "^AssignVariableOp_100" input: "^AssignVariableOp_101" input: "^AssignVariableOp_102" input: "^AssignVariableOp_103" input: "^AssignVariableOp_104" input: "^AssignVariableOp_105" input: "^AssignVariableOp_106" input: "^AssignVariableOp_107" input: "^AssignVariableOp_108" input: "^AssignVariableOp_109" input: "^AssignVariableOp_11" input: "^AssignVariableOp_110" input: "^AssignVariableOp_111" input: "^AssignVariableOp_112" input: "^AssignVariableOp_113" input: "^AssignVariableOp_114" input: "^AssignVariableOp_115" input: "^AssignVariableOp_116" input: "^AssignVariableOp_117" input: "^AssignVariableOp_118" input: "^AssignVariableOp_119" input: "^AssignVariableOp_12" input: "^AssignVariableOp_120" input: "^AssignVariableOp_121" input: "^AssignVariableOp_122" input: "^AssignVariableOp_123" input: "^AssignVariableOp_124" input: "^AssignVariableOp_125" input: "^AssignVariableOp_126" input: "^AssignVariableOp_127" input: "^AssignVariableOp_128" input: "^AssignVariableOp_129" input: "^AssignVariableOp_13" input: "^AssignVariableOp_130" input: "^AssignVariableOp_131" input: "^AssignVariableOp_132" input: "^AssignVariableOp_133" input: "^AssignVariableOp_134" input: "^AssignVariableOp_135" input: "^AssignVariableOp_136" input: "^AssignVariableOp_137" input: "^AssignVariableOp_138" input: "^AssignVariableOp_139" input: "^AssignVariableOp_14" input: "^AssignVariableOp_140" input: "^AssignVariableOp_141" input: "^AssignVariableOp_142" input: "^AssignVariableOp_143" input: "^AssignVariableOp_144" input: "^AssignVariableOp_145" input: "^AssignVariableOp_146" input: "^AssignVariableOp_147" input: "^AssignVariableOp_148" input: "^AssignVariableOp_15" input: "^AssignVariableOp_16" input: "^AssignVariableOp_17" input: "^AssignVariableOp_18" input: "^AssignVariableOp_19" input: "^AssignVariableOp_2" input: "^AssignVariableOp_20" input: "^AssignVariableOp_21" input: "^AssignVariableOp_22" input: "^AssignVariableOp_23" input: "^AssignVariableOp_24" input: "^AssignVariableOp_25" input: "^AssignVariableOp_26" input: "^AssignVariableOp_27" input: "^AssignVariableOp_28" input: "^AssignVariableOp_29" input: "^AssignVariableOp_3" input: "^AssignVariableOp_30" input: "^AssignVariableOp_31" input: "^AssignVariableOp_32" input: "^AssignVariableOp_33" input: "^AssignVariableOp_34" input: "^AssignVariableOp_35" input: "^AssignVariableOp_36" input: "^AssignVariableOp_37" input: "^AssignVariableOp_38" input: "^AssignVariableOp_39" input: "^AssignVariableOp_4" input: "^AssignVariableOp_40" input: "^AssignVariableOp_41" input: "^AssignVariableOp_42" input: "^AssignVariableOp_43" input: "^AssignVariableOp_44" input: "^AssignVariableOp_45" input: "^AssignVariableOp_46" input: "^AssignVariableOp_47" input: "^AssignVariableOp_48" input: "^AssignVariableOp_49" input: "^AssignVariableOp_5" input: "^AssignVariableOp_50" input: "^AssignVariableOp_51" input: "^AssignVariableOp_52" input: "^AssignVariableOp_53" input: "^AssignVariableOp_54" input: "^AssignVariableOp_55" input: "^AssignVariableOp_56" input: "^AssignVariableOp_57" input: "^AssignVariableOp_58" input: "^AssignVariableOp_59" input: "^AssignVariableOp_6" input: "^AssignVariableOp_60" input: "^AssignVariableOp_61" input: "^AssignVariableOp_62" input: "^AssignVariableOp_63" input: "^AssignVariableOp_64" input: "^AssignVariableOp_65" input: "^AssignVariableOp_66" input: "^AssignVariableOp_67" input: "^AssignVariableOp_68" input: "^AssignVariableOp_69" input: "^AssignVariableOp_7" input: "^AssignVariableOp_70" input: "^AssignVariableOp_71" input: "^AssignVariableOp_72" input: "^AssignVariableOp_73" input: "^AssignVariableOp_74" input: "^AssignVariableOp_75" input: "^AssignVariableOp_76" input: "^AssignVariableOp_77" input: "^AssignVariableOp_78" input: "^AssignVariableOp_79" input: "^AssignVariableOp_8" input: "^AssignVariableOp_80" input: "^AssignVariableOp_81" input: "^AssignVariableOp_82" input: "^AssignVariableOp_83" input: "^AssignVariableOp_84" input: "^AssignVariableOp_85" input: "^AssignVariableOp_86" input: "^AssignVariableOp_87" input: "^AssignVariableOp_88" input: "^AssignVariableOp_89" input: "^AssignVariableOp_9" input: "^AssignVariableOp_90" input: "^AssignVariableOp_91" input: "^AssignVariableOp_92" input: "^AssignVariableOp_93" input: "^AssignVariableOp_94" input: "^AssignVariableOp_95" input: "^AssignVariableOp_96" input: "^AssignVariableOp_97" input: "^AssignVariableOp_98" input: "^AssignVariableOp_99" input: "^bert/embeddings/layer_normalization/beta/Assign" input: "^bert/embeddings/layer_normalization/beta/adam_m/Assign" input: "^bert/embeddings/layer_normalization/beta/adam_v/Assign" input: "^bert/embeddings/layer_normalization/gamma/Assign" input: "^bert/embeddings/layer_normalization/gamma/adam_m/Assign" input: "^bert/embeddings/layer_normalization/gamma/adam_v/Assign" input: "^bert/embeddings/position_embeddings/adam_m/Assign" input: "^bert/embeddings/position_embeddings/adam_v/Assign" input: "^bert/embeddings/token_type_embeddings/adam_m/Assign" input: "^bert/embeddings/token_type_embeddings/adam_v/Assign" input: "^bert/embeddings/word_embeddings/adam_m/Assign" input: "^bert/embeddings/word_embeddings/adam_v/Assign" input: "^bert/encoder/layer_0/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_0/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_0/attention/output/layer_normalization_1/beta/Assign" input: "^bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m/Assign" input: "^bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v/Assign" input: "^bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/Assign" input: "^bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m/Assign" input: "^bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v/Assign" input: "^bert/encoder/layer_0/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_0/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_0/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_0/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_0/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_0/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_0/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_0/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_0/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_0/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_0/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_0/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_0/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_0/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_0/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_0/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_0/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_0/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_0/output/layer_normalization_2/beta/Assign" input: "^bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m/Assign" input: "^bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v/Assign" input: "^bert/encoder/layer_0/output/layer_normalization_2/gamma/Assign" input: "^bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m/Assign" input: "^bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v/Assign" input: "^bert/encoder/layer_1/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_1/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_1/attention/output/layer_normalization_3/beta/Assign" input: "^bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m/Assign" input: "^bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v/Assign" input: "^bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/Assign" input: "^bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m/Assign" input: "^bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v/Assign" input: "^bert/encoder/layer_1/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_1/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_1/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_1/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_1/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_1/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_1/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_1/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_1/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_1/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_1/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_1/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_1/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_1/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_1/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_1/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_1/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_1/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_1/output/layer_normalization_4/beta/Assign" input: "^bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m/Assign" input: "^bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v/Assign" input: "^bert/encoder/layer_1/output/layer_normalization_4/gamma/Assign" input: "^bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m/Assign" input: "^bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v/Assign" input: "^bert/encoder/layer_10/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_10/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_10/attention/output/layer_normalization_21/beta/Assign" input: "^bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m/Assign" input: "^bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v/Assign" input: "^bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/Assign" input: "^bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m/Assign" input: "^bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v/Assign" input: "^bert/encoder/layer_10/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_10/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_10/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_10/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_10/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_10/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_10/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_10/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_10/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_10/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_10/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_10/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_10/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_10/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_10/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_10/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_10/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_10/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_10/output/layer_normalization_22/beta/Assign" input: "^bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m/Assign" input: "^bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v/Assign" input: "^bert/encoder/layer_10/output/layer_normalization_22/gamma/Assign" input: "^bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m/Assign" input: "^bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v/Assign" input: "^bert/encoder/layer_11/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_11/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_11/attention/output/layer_normalization_23/beta/Assign" input: "^bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m/Assign" input: "^bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v/Assign" input: "^bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/Assign" input: "^bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m/Assign" input: "^bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v/Assign" input: "^bert/encoder/layer_11/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_11/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_11/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_11/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_11/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_11/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_11/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_11/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_11/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_11/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_11/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_11/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_11/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_11/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_11/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_11/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_11/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_11/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_11/output/layer_normalization_24/beta/Assign" input: "^bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m/Assign" input: "^bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v/Assign" input: "^bert/encoder/layer_11/output/layer_normalization_24/gamma/Assign" input: "^bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m/Assign" input: "^bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v/Assign" input: "^bert/encoder/layer_2/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_2/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_2/attention/output/layer_normalization_5/beta/Assign" input: "^bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m/Assign" input: "^bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v/Assign" input: "^bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/Assign" input: "^bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m/Assign" input: "^bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v/Assign" input: "^bert/encoder/layer_2/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_2/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_2/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_2/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_2/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_2/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_2/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_2/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_2/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_2/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_2/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_2/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_2/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_2/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_2/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_2/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_2/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_2/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_2/output/layer_normalization_6/beta/Assign" input: "^bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m/Assign" input: "^bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v/Assign" input: "^bert/encoder/layer_2/output/layer_normalization_6/gamma/Assign" input: "^bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m/Assign" input: "^bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v/Assign" input: "^bert/encoder/layer_3/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_3/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_3/attention/output/layer_normalization_7/beta/Assign" input: "^bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m/Assign" input: "^bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v/Assign" input: "^bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/Assign" input: "^bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m/Assign" input: "^bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v/Assign" input: "^bert/encoder/layer_3/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_3/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_3/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_3/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_3/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_3/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_3/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_3/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_3/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_3/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_3/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_3/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_3/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_3/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_3/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_3/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_3/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_3/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_3/output/layer_normalization_8/beta/Assign" input: "^bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m/Assign" input: "^bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v/Assign" input: "^bert/encoder/layer_3/output/layer_normalization_8/gamma/Assign" input: "^bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m/Assign" input: "^bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v/Assign" input: "^bert/encoder/layer_4/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_4/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_4/attention/output/layer_normalization_9/beta/Assign" input: "^bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m/Assign" input: "^bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v/Assign" input: "^bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/Assign" input: "^bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m/Assign" input: "^bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v/Assign" input: "^bert/encoder/layer_4/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_4/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_4/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_4/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_4/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_4/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_4/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_4/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_4/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_4/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_4/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_4/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_4/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_4/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_4/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_4/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_4/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_4/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_4/output/layer_normalization_10/beta/Assign" input: "^bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m/Assign" input: "^bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v/Assign" input: "^bert/encoder/layer_4/output/layer_normalization_10/gamma/Assign" input: "^bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m/Assign" input: "^bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v/Assign" input: "^bert/encoder/layer_5/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_5/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_5/attention/output/layer_normalization_11/beta/Assign" input: "^bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m/Assign" input: "^bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v/Assign" input: "^bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/Assign" input: "^bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m/Assign" input: "^bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v/Assign" input: "^bert/encoder/layer_5/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_5/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_5/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_5/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_5/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_5/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_5/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_5/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_5/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_5/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_5/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_5/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_5/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_5/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_5/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_5/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_5/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_5/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_5/output/layer_normalization_12/beta/Assign" input: "^bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m/Assign" input: "^bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v/Assign" input: "^bert/encoder/layer_5/output/layer_normalization_12/gamma/Assign" input: "^bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m/Assign" input: "^bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v/Assign" input: "^bert/encoder/layer_6/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_6/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_6/attention/output/layer_normalization_13/beta/Assign" input: "^bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m/Assign" input: "^bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v/Assign" input: "^bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/Assign" input: "^bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m/Assign" input: "^bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v/Assign" input: "^bert/encoder/layer_6/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_6/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_6/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_6/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_6/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_6/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_6/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_6/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_6/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_6/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_6/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_6/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_6/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_6/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_6/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_6/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_6/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_6/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_6/output/layer_normalization_14/beta/Assign" input: "^bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m/Assign" input: "^bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v/Assign" input: "^bert/encoder/layer_6/output/layer_normalization_14/gamma/Assign" input: "^bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m/Assign" input: "^bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v/Assign" input: "^bert/encoder/layer_7/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_7/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_7/attention/output/layer_normalization_15/beta/Assign" input: "^bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m/Assign" input: "^bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v/Assign" input: "^bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/Assign" input: "^bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m/Assign" input: "^bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v/Assign" input: "^bert/encoder/layer_7/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_7/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_7/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_7/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_7/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_7/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_7/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_7/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_7/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_7/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_7/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_7/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_7/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_7/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_7/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_7/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_7/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_7/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_7/output/layer_normalization_16/beta/Assign" input: "^bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m/Assign" input: "^bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v/Assign" input: "^bert/encoder/layer_7/output/layer_normalization_16/gamma/Assign" input: "^bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m/Assign" input: "^bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v/Assign" input: "^bert/encoder/layer_8/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_8/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_8/attention/output/layer_normalization_17/beta/Assign" input: "^bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m/Assign" input: "^bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v/Assign" input: "^bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/Assign" input: "^bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m/Assign" input: "^bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v/Assign" input: "^bert/encoder/layer_8/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_8/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_8/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_8/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_8/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_8/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_8/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_8/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_8/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_8/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_8/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_8/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_8/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_8/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_8/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_8/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_8/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_8/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_8/output/layer_normalization_18/beta/Assign" input: "^bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m/Assign" input: "^bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v/Assign" input: "^bert/encoder/layer_8/output/layer_normalization_18/gamma/Assign" input: "^bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m/Assign" input: "^bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v/Assign" input: "^bert/encoder/layer_9/attention/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_9/attention/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_9/attention/output/layer_normalization_19/beta/Assign" input: "^bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m/Assign" input: "^bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v/Assign" input: "^bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/Assign" input: "^bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m/Assign" input: "^bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v/Assign" input: "^bert/encoder/layer_9/attention/self/key/bias/adam_m/Assign" input: "^bert/encoder/layer_9/attention/self/key/bias/adam_v/Assign" input: "^bert/encoder/layer_9/attention/self/key/kernel/adam_m/Assign" input: "^bert/encoder/layer_9/attention/self/key/kernel/adam_v/Assign" input: "^bert/encoder/layer_9/attention/self/query/bias/adam_m/Assign" input: "^bert/encoder/layer_9/attention/self/query/bias/adam_v/Assign" input: "^bert/encoder/layer_9/attention/self/query/kernel/adam_m/Assign" input: "^bert/encoder/layer_9/attention/self/query/kernel/adam_v/Assign" input: "^bert/encoder/layer_9/attention/self/value/bias/adam_m/Assign" input: "^bert/encoder/layer_9/attention/self/value/bias/adam_v/Assign" input: "^bert/encoder/layer_9/attention/self/value/kernel/adam_m/Assign" input: "^bert/encoder/layer_9/attention/self/value/kernel/adam_v/Assign" input: "^bert/encoder/layer_9/intermediate/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_9/intermediate/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_9/output/dense/bias/adam_m/Assign" input: "^bert/encoder/layer_9/output/dense/bias/adam_v/Assign" input: "^bert/encoder/layer_9/output/dense/kernel/adam_m/Assign" input: "^bert/encoder/layer_9/output/dense/kernel/adam_v/Assign" input: "^bert/encoder/layer_9/output/layer_normalization_20/beta/Assign" input: "^bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m/Assign" input: "^bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v/Assign" input: "^bert/encoder/layer_9/output/layer_normalization_20/gamma/Assign" input: "^bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m/Assign" input: "^bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v/Assign" input: "^bert/pooler/dense/bias/adam_m/Assign" input: "^bert/pooler/dense/bias/adam_v/Assign" input: "^bert/pooler/dense/kernel/adam_m/Assign" input: "^bert/pooler/dense/kernel/adam_v/Assign" input: "^global_step/Assign" input: "^output_bias/Assign" input: "^output_bias/adam_m/Assign" input: "^output_bias/adam_v/Assign" input: "^output_weights/Assign" input: "^output_weights/adam_m/Assign" input: "^output_weights/adam_v/Assign" attr { key: "_has_manual_control_dependencies" value { b: true } } } node { name: "init_1" op: "NoOp" attr { key: "_has_manual_control_dependencies" value { b: true } } } node { name: "group_deps_2" op: "NoOp" input: "^init" input: "^init_1" } node { name: "report_uninitialized_variables/VarIsInitializedOp" op: "VarIsInitializedOp" input: "global_step" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_1" op: "VarIsInitializedOp" input: "bert/embeddings/word_embeddings" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_2" op: "VarIsInitializedOp" input: "bert/embeddings/token_type_embeddings" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_3" op: "VarIsInitializedOp" input: "bert/embeddings/position_embeddings" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_4" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_5" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_6" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_7" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_8" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_9" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_10" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_11" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_12" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_13" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_14" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_15" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_16" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_17" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_18" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_19" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_20" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_21" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_22" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_23" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_24" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_25" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_26" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_27" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_28" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_29" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_30" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_31" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_32" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_33" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_34" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_35" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_36" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_37" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_38" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_39" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_40" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_41" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_42" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_43" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_44" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_45" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_46" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_47" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_48" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_49" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_50" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_51" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_52" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_53" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_54" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_55" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_56" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_57" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_58" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_59" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_60" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_61" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_62" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_63" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_64" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_65" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_66" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_67" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_68" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_69" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_70" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_71" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_72" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_73" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_74" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_75" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_76" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_77" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_78" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_79" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_80" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_81" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_82" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_83" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_84" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_85" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_86" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_87" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_88" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_89" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_90" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_91" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_92" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_93" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_94" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_95" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_96" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_97" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_98" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_99" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_100" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_101" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_102" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_103" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_104" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_105" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_106" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_107" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_108" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_109" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_110" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_111" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_112" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_113" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_114" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_115" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_116" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_117" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_118" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_119" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_120" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_121" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_122" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_123" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_124" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_125" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_126" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_127" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_128" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_129" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_130" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_131" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_132" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_133" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_134" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_135" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_136" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_137" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_138" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_139" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_140" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_141" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_142" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_143" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_144" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_145" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_146" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_147" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_148" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_149" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_150" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_151" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_152" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_153" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_154" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_155" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_156" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_157" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_158" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_159" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_160" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_161" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_162" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_163" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_164" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_165" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_166" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_167" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_168" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_169" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_170" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_171" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_172" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_173" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_174" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_175" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_176" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_177" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_178" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_179" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_180" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_181" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_182" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_183" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_184" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_185" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_186" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_187" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_188" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_189" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_190" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_191" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_192" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_193" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_194" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_195" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_196" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_197" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_198" op: "VarIsInitializedOp" input: "bert/pooler/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_199" op: "VarIsInitializedOp" input: "bert/pooler/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_200" op: "VarIsInitializedOp" input: "output_weights" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_201" op: "VarIsInitializedOp" input: "output_bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_202" op: "VarIsInitializedOp" input: "bert/embeddings/word_embeddings/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_203" op: "VarIsInitializedOp" input: "bert/embeddings/word_embeddings/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_204" op: "VarIsInitializedOp" input: "bert/embeddings/token_type_embeddings/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_205" op: "VarIsInitializedOp" input: "bert/embeddings/token_type_embeddings/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_206" op: "VarIsInitializedOp" input: "bert/embeddings/position_embeddings/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_207" op: "VarIsInitializedOp" input: "bert/embeddings/position_embeddings/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_208" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_209" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_210" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_211" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_212" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_213" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_214" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_215" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_216" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_217" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_218" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_219" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_220" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_221" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_222" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_223" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_224" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_225" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_226" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_227" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_228" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_229" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_230" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_231" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_232" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_233" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_234" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_235" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_236" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_237" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_238" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_239" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_240" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_241" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_242" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_243" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_244" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_245" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_246" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_247" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_248" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_249" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_250" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_251" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_252" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_253" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_254" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_255" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_256" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_257" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_258" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_259" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_260" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_261" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_262" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_263" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_264" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_265" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_266" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_267" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_268" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_269" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_270" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_271" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_272" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_273" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_274" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_275" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_276" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_277" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_278" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_279" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_280" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_281" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_282" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_283" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_284" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_285" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_286" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_287" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_288" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_289" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_290" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_291" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_292" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_293" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_294" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_295" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_296" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_297" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_298" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_299" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_300" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_301" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_302" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_303" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_304" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_305" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_306" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_307" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_308" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_309" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_310" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_311" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_312" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_313" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_314" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_315" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_316" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_317" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_318" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_319" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_320" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_321" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_322" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_323" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_324" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_325" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_326" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_327" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_328" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_329" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_330" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_331" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_332" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_333" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_334" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_335" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_336" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_337" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_338" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_339" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_340" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_341" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_342" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_343" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_344" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_345" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_346" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_347" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_348" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_349" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_350" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_351" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_352" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_353" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_354" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_355" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_356" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_357" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_358" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_359" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_360" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_361" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_362" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_363" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_364" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_365" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_366" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_367" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_368" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_369" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_370" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_371" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_372" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_373" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_374" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_375" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_376" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_377" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_378" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_379" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_380" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_381" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_382" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_383" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_384" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_385" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_386" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_387" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_388" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_389" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_390" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_391" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_392" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_393" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_394" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_395" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_396" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_397" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_398" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_399" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_400" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_401" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_402" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_403" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_404" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_405" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_406" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_407" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_408" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_409" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_410" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_411" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_412" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_413" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_414" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_415" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_416" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_417" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_418" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_419" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_420" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_421" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_422" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_423" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_424" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_425" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_426" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_427" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_428" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_429" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_430" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_431" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_432" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_433" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_434" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_435" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_436" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_437" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_438" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_439" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_440" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_441" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_442" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_443" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_444" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_445" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_446" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_447" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_448" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_449" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_450" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_451" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_452" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_453" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_454" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_455" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_456" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_457" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_458" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_459" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_460" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_461" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_462" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_463" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_464" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_465" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_466" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_467" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_468" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_469" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_470" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_471" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_472" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_473" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_474" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_475" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_476" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_477" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_478" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_479" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_480" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_481" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_482" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_483" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_484" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_485" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_486" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_487" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_488" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_489" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_490" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_491" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_492" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_493" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_494" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_495" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_496" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_497" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_498" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_499" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_500" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_501" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_502" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_503" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_504" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_505" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_506" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_507" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_508" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_509" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_510" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_511" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_512" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_513" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_514" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_515" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_516" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_517" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_518" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_519" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_520" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_521" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_522" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_523" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_524" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_525" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_526" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_527" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_528" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_529" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_530" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_531" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_532" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_533" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_534" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_535" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_536" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_537" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_538" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_539" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_540" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_541" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_542" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_543" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_544" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_545" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_546" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_547" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_548" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_549" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_550" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_551" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_552" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_553" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_554" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_555" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_556" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_557" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_558" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_559" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_560" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_561" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_562" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_563" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_564" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_565" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_566" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_567" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_568" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_569" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_570" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_571" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_572" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_573" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_574" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_575" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_576" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_577" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_578" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_579" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_580" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_581" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_582" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_583" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_584" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_585" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_586" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_587" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_588" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_589" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_590" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_591" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_592" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_593" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_594" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_595" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_596" op: "VarIsInitializedOp" input: "bert/pooler/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_597" op: "VarIsInitializedOp" input: "bert/pooler/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_598" op: "VarIsInitializedOp" input: "bert/pooler/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_599" op: "VarIsInitializedOp" input: "bert/pooler/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_600" op: "VarIsInitializedOp" input: "output_weights/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_601" op: "VarIsInitializedOp" input: "output_weights/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_602" op: "VarIsInitializedOp" input: "output_bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/VarIsInitializedOp_603" op: "VarIsInitializedOp" input: "output_bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables/stack" op: "Pack" input: "report_uninitialized_variables/VarIsInitializedOp" input: "report_uninitialized_variables/VarIsInitializedOp_1" input: "report_uninitialized_variables/VarIsInitializedOp_2" input: "report_uninitialized_variables/VarIsInitializedOp_3" input: "report_uninitialized_variables/VarIsInitializedOp_4" input: "report_uninitialized_variables/VarIsInitializedOp_5" input: "report_uninitialized_variables/VarIsInitializedOp_6" input: "report_uninitialized_variables/VarIsInitializedOp_7" input: "report_uninitialized_variables/VarIsInitializedOp_8" input: "report_uninitialized_variables/VarIsInitializedOp_9" input: "report_uninitialized_variables/VarIsInitializedOp_10" input: "report_uninitialized_variables/VarIsInitializedOp_11" input: "report_uninitialized_variables/VarIsInitializedOp_12" input: "report_uninitialized_variables/VarIsInitializedOp_13" input: "report_uninitialized_variables/VarIsInitializedOp_14" input: "report_uninitialized_variables/VarIsInitializedOp_15" input: "report_uninitialized_variables/VarIsInitializedOp_16" input: "report_uninitialized_variables/VarIsInitializedOp_17" input: "report_uninitialized_variables/VarIsInitializedOp_18" input: "report_uninitialized_variables/VarIsInitializedOp_19" input: "report_uninitialized_variables/VarIsInitializedOp_20" input: "report_uninitialized_variables/VarIsInitializedOp_21" input: "report_uninitialized_variables/VarIsInitializedOp_22" input: "report_uninitialized_variables/VarIsInitializedOp_23" input: "report_uninitialized_variables/VarIsInitializedOp_24" input: "report_uninitialized_variables/VarIsInitializedOp_25" input: "report_uninitialized_variables/VarIsInitializedOp_26" input: "report_uninitialized_variables/VarIsInitializedOp_27" input: "report_uninitialized_variables/VarIsInitializedOp_28" input: "report_uninitialized_variables/VarIsInitializedOp_29" input: "report_uninitialized_variables/VarIsInitializedOp_30" input: "report_uninitialized_variables/VarIsInitializedOp_31" input: "report_uninitialized_variables/VarIsInitializedOp_32" input: "report_uninitialized_variables/VarIsInitializedOp_33" input: "report_uninitialized_variables/VarIsInitializedOp_34" input: "report_uninitialized_variables/VarIsInitializedOp_35" input: "report_uninitialized_variables/VarIsInitializedOp_36" input: "report_uninitialized_variables/VarIsInitializedOp_37" input: "report_uninitialized_variables/VarIsInitializedOp_38" input: "report_uninitialized_variables/VarIsInitializedOp_39" input: "report_uninitialized_variables/VarIsInitializedOp_40" input: "report_uninitialized_variables/VarIsInitializedOp_41" input: "report_uninitialized_variables/VarIsInitializedOp_42" input: "report_uninitialized_variables/VarIsInitializedOp_43" input: "report_uninitialized_variables/VarIsInitializedOp_44" input: "report_uninitialized_variables/VarIsInitializedOp_45" input: "report_uninitialized_variables/VarIsInitializedOp_46" input: "report_uninitialized_variables/VarIsInitializedOp_47" input: "report_uninitialized_variables/VarIsInitializedOp_48" input: "report_uninitialized_variables/VarIsInitializedOp_49" input: "report_uninitialized_variables/VarIsInitializedOp_50" input: "report_uninitialized_variables/VarIsInitializedOp_51" input: "report_uninitialized_variables/VarIsInitializedOp_52" input: "report_uninitialized_variables/VarIsInitializedOp_53" input: "report_uninitialized_variables/VarIsInitializedOp_54" input: "report_uninitialized_variables/VarIsInitializedOp_55" input: "report_uninitialized_variables/VarIsInitializedOp_56" input: "report_uninitialized_variables/VarIsInitializedOp_57" input: "report_uninitialized_variables/VarIsInitializedOp_58" input: "report_uninitialized_variables/VarIsInitializedOp_59" input: "report_uninitialized_variables/VarIsInitializedOp_60" input: "report_uninitialized_variables/VarIsInitializedOp_61" input: "report_uninitialized_variables/VarIsInitializedOp_62" input: "report_uninitialized_variables/VarIsInitializedOp_63" input: "report_uninitialized_variables/VarIsInitializedOp_64" input: "report_uninitialized_variables/VarIsInitializedOp_65" input: "report_uninitialized_variables/VarIsInitializedOp_66" input: "report_uninitialized_variables/VarIsInitializedOp_67" input: "report_uninitialized_variables/VarIsInitializedOp_68" input: "report_uninitialized_variables/VarIsInitializedOp_69" input: "report_uninitialized_variables/VarIsInitializedOp_70" input: "report_uninitialized_variables/VarIsInitializedOp_71" input: "report_uninitialized_variables/VarIsInitializedOp_72" input: "report_uninitialized_variables/VarIsInitializedOp_73" input: "report_uninitialized_variables/VarIsInitializedOp_74" input: "report_uninitialized_variables/VarIsInitializedOp_75" input: "report_uninitialized_variables/VarIsInitializedOp_76" input: "report_uninitialized_variables/VarIsInitializedOp_77" input: "report_uninitialized_variables/VarIsInitializedOp_78" input: "report_uninitialized_variables/VarIsInitializedOp_79" input: "report_uninitialized_variables/VarIsInitializedOp_80" input: "report_uninitialized_variables/VarIsInitializedOp_81" input: "report_uninitialized_variables/VarIsInitializedOp_82" input: "report_uninitialized_variables/VarIsInitializedOp_83" input: "report_uninitialized_variables/VarIsInitializedOp_84" input: "report_uninitialized_variables/VarIsInitializedOp_85" input: "report_uninitialized_variables/VarIsInitializedOp_86" input: "report_uninitialized_variables/VarIsInitializedOp_87" input: "report_uninitialized_variables/VarIsInitializedOp_88" input: "report_uninitialized_variables/VarIsInitializedOp_89" input: "report_uninitialized_variables/VarIsInitializedOp_90" input: "report_uninitialized_variables/VarIsInitializedOp_91" input: "report_uninitialized_variables/VarIsInitializedOp_92" input: "report_uninitialized_variables/VarIsInitializedOp_93" input: "report_uninitialized_variables/VarIsInitializedOp_94" input: "report_uninitialized_variables/VarIsInitializedOp_95" input: "report_uninitialized_variables/VarIsInitializedOp_96" input: "report_uninitialized_variables/VarIsInitializedOp_97" input: "report_uninitialized_variables/VarIsInitializedOp_98" input: "report_uninitialized_variables/VarIsInitializedOp_99" input: "report_uninitialized_variables/VarIsInitializedOp_100" input: "report_uninitialized_variables/VarIsInitializedOp_101" input: "report_uninitialized_variables/VarIsInitializedOp_102" input: "report_uninitialized_variables/VarIsInitializedOp_103" input: "report_uninitialized_variables/VarIsInitializedOp_104" input: "report_uninitialized_variables/VarIsInitializedOp_105" input: "report_uninitialized_variables/VarIsInitializedOp_106" input: "report_uninitialized_variables/VarIsInitializedOp_107" input: "report_uninitialized_variables/VarIsInitializedOp_108" input: "report_uninitialized_variables/VarIsInitializedOp_109" input: "report_uninitialized_variables/VarIsInitializedOp_110" input: "report_uninitialized_variables/VarIsInitializedOp_111" input: "report_uninitialized_variables/VarIsInitializedOp_112" input: "report_uninitialized_variables/VarIsInitializedOp_113" input: "report_uninitialized_variables/VarIsInitializedOp_114" input: "report_uninitialized_variables/VarIsInitializedOp_115" input: "report_uninitialized_variables/VarIsInitializedOp_116" input: "report_uninitialized_variables/VarIsInitializedOp_117" input: "report_uninitialized_variables/VarIsInitializedOp_118" input: "report_uninitialized_variables/VarIsInitializedOp_119" input: "report_uninitialized_variables/VarIsInitializedOp_120" input: "report_uninitialized_variables/VarIsInitializedOp_121" input: "report_uninitialized_variables/VarIsInitializedOp_122" input: "report_uninitialized_variables/VarIsInitializedOp_123" input: "report_uninitialized_variables/VarIsInitializedOp_124" input: "report_uninitialized_variables/VarIsInitializedOp_125" input: "report_uninitialized_variables/VarIsInitializedOp_126" input: "report_uninitialized_variables/VarIsInitializedOp_127" input: "report_uninitialized_variables/VarIsInitializedOp_128" input: "report_uninitialized_variables/VarIsInitializedOp_129" input: "report_uninitialized_variables/VarIsInitializedOp_130" input: "report_uninitialized_variables/VarIsInitializedOp_131" input: "report_uninitialized_variables/VarIsInitializedOp_132" input: "report_uninitialized_variables/VarIsInitializedOp_133" input: "report_uninitialized_variables/VarIsInitializedOp_134" input: "report_uninitialized_variables/VarIsInitializedOp_135" input: "report_uninitialized_variables/VarIsInitializedOp_136" input: "report_uninitialized_variables/VarIsInitializedOp_137" input: "report_uninitialized_variables/VarIsInitializedOp_138" input: "report_uninitialized_variables/VarIsInitializedOp_139" input: "report_uninitialized_variables/VarIsInitializedOp_140" input: "report_uninitialized_variables/VarIsInitializedOp_141" input: "report_uninitialized_variables/VarIsInitializedOp_142" input: "report_uninitialized_variables/VarIsInitializedOp_143" input: "report_uninitialized_variables/VarIsInitializedOp_144" input: "report_uninitialized_variables/VarIsInitializedOp_145" input: "report_uninitialized_variables/VarIsInitializedOp_146" input: "report_uninitialized_variables/VarIsInitializedOp_147" input: "report_uninitialized_variables/VarIsInitializedOp_148" input: "report_uninitialized_variables/VarIsInitializedOp_149" input: "report_uninitialized_variables/VarIsInitializedOp_150" input: "report_uninitialized_variables/VarIsInitializedOp_151" input: "report_uninitialized_variables/VarIsInitializedOp_152" input: "report_uninitialized_variables/VarIsInitializedOp_153" input: "report_uninitialized_variables/VarIsInitializedOp_154" input: "report_uninitialized_variables/VarIsInitializedOp_155" input: "report_uninitialized_variables/VarIsInitializedOp_156" input: "report_uninitialized_variables/VarIsInitializedOp_157" input: "report_uninitialized_variables/VarIsInitializedOp_158" input: "report_uninitialized_variables/VarIsInitializedOp_159" input: "report_uninitialized_variables/VarIsInitializedOp_160" input: "report_uninitialized_variables/VarIsInitializedOp_161" input: "report_uninitialized_variables/VarIsInitializedOp_162" input: "report_uninitialized_variables/VarIsInitializedOp_163" input: "report_uninitialized_variables/VarIsInitializedOp_164" input: "report_uninitialized_variables/VarIsInitializedOp_165" input: "report_uninitialized_variables/VarIsInitializedOp_166" input: "report_uninitialized_variables/VarIsInitializedOp_167" input: "report_uninitialized_variables/VarIsInitializedOp_168" input: "report_uninitialized_variables/VarIsInitializedOp_169" input: "report_uninitialized_variables/VarIsInitializedOp_170" input: "report_uninitialized_variables/VarIsInitializedOp_171" input: "report_uninitialized_variables/VarIsInitializedOp_172" input: "report_uninitialized_variables/VarIsInitializedOp_173" input: "report_uninitialized_variables/VarIsInitializedOp_174" input: "report_uninitialized_variables/VarIsInitializedOp_175" input: "report_uninitialized_variables/VarIsInitializedOp_176" input: "report_uninitialized_variables/VarIsInitializedOp_177" input: "report_uninitialized_variables/VarIsInitializedOp_178" input: "report_uninitialized_variables/VarIsInitializedOp_179" input: "report_uninitialized_variables/VarIsInitializedOp_180" input: "report_uninitialized_variables/VarIsInitializedOp_181" input: "report_uninitialized_variables/VarIsInitializedOp_182" input: "report_uninitialized_variables/VarIsInitializedOp_183" input: "report_uninitialized_variables/VarIsInitializedOp_184" input: "report_uninitialized_variables/VarIsInitializedOp_185" input: "report_uninitialized_variables/VarIsInitializedOp_186" input: "report_uninitialized_variables/VarIsInitializedOp_187" input: "report_uninitialized_variables/VarIsInitializedOp_188" input: "report_uninitialized_variables/VarIsInitializedOp_189" input: "report_uninitialized_variables/VarIsInitializedOp_190" input: "report_uninitialized_variables/VarIsInitializedOp_191" input: "report_uninitialized_variables/VarIsInitializedOp_192" input: "report_uninitialized_variables/VarIsInitializedOp_193" input: "report_uninitialized_variables/VarIsInitializedOp_194" input: "report_uninitialized_variables/VarIsInitializedOp_195" input: "report_uninitialized_variables/VarIsInitializedOp_196" input: "report_uninitialized_variables/VarIsInitializedOp_197" input: "report_uninitialized_variables/VarIsInitializedOp_198" input: "report_uninitialized_variables/VarIsInitializedOp_199" input: "report_uninitialized_variables/VarIsInitializedOp_200" input: "report_uninitialized_variables/VarIsInitializedOp_201" input: "report_uninitialized_variables/VarIsInitializedOp_202" input: "report_uninitialized_variables/VarIsInitializedOp_203" input: "report_uninitialized_variables/VarIsInitializedOp_204" input: "report_uninitialized_variables/VarIsInitializedOp_205" input: "report_uninitialized_variables/VarIsInitializedOp_206" input: "report_uninitialized_variables/VarIsInitializedOp_207" input: "report_uninitialized_variables/VarIsInitializedOp_208" input: "report_uninitialized_variables/VarIsInitializedOp_209" input: "report_uninitialized_variables/VarIsInitializedOp_210" input: "report_uninitialized_variables/VarIsInitializedOp_211" input: "report_uninitialized_variables/VarIsInitializedOp_212" input: "report_uninitialized_variables/VarIsInitializedOp_213" input: "report_uninitialized_variables/VarIsInitializedOp_214" input: "report_uninitialized_variables/VarIsInitializedOp_215" input: "report_uninitialized_variables/VarIsInitializedOp_216" input: "report_uninitialized_variables/VarIsInitializedOp_217" input: "report_uninitialized_variables/VarIsInitializedOp_218" input: "report_uninitialized_variables/VarIsInitializedOp_219" input: "report_uninitialized_variables/VarIsInitializedOp_220" input: "report_uninitialized_variables/VarIsInitializedOp_221" input: "report_uninitialized_variables/VarIsInitializedOp_222" input: "report_uninitialized_variables/VarIsInitializedOp_223" input: "report_uninitialized_variables/VarIsInitializedOp_224" input: "report_uninitialized_variables/VarIsInitializedOp_225" input: "report_uninitialized_variables/VarIsInitializedOp_226" input: "report_uninitialized_variables/VarIsInitializedOp_227" input: "report_uninitialized_variables/VarIsInitializedOp_228" input: "report_uninitialized_variables/VarIsInitializedOp_229" input: "report_uninitialized_variables/VarIsInitializedOp_230" input: "report_uninitialized_variables/VarIsInitializedOp_231" input: "report_uninitialized_variables/VarIsInitializedOp_232" input: "report_uninitialized_variables/VarIsInitializedOp_233" input: "report_uninitialized_variables/VarIsInitializedOp_234" input: "report_uninitialized_variables/VarIsInitializedOp_235" input: "report_uninitialized_variables/VarIsInitializedOp_236" input: "report_uninitialized_variables/VarIsInitializedOp_237" input: "report_uninitialized_variables/VarIsInitializedOp_238" input: "report_uninitialized_variables/VarIsInitializedOp_239" input: "report_uninitialized_variables/VarIsInitializedOp_240" input: "report_uninitialized_variables/VarIsInitializedOp_241" input: "report_uninitialized_variables/VarIsInitializedOp_242" input: "report_uninitialized_variables/VarIsInitializedOp_243" input: "report_uninitialized_variables/VarIsInitializedOp_244" input: "report_uninitialized_variables/VarIsInitializedOp_245" input: "report_uninitialized_variables/VarIsInitializedOp_246" input: "report_uninitialized_variables/VarIsInitializedOp_247" input: "report_uninitialized_variables/VarIsInitializedOp_248" input: "report_uninitialized_variables/VarIsInitializedOp_249" input: "report_uninitialized_variables/VarIsInitializedOp_250" input: "report_uninitialized_variables/VarIsInitializedOp_251" input: "report_uninitialized_variables/VarIsInitializedOp_252" input: "report_uninitialized_variables/VarIsInitializedOp_253" input: "report_uninitialized_variables/VarIsInitializedOp_254" input: "report_uninitialized_variables/VarIsInitializedOp_255" input: "report_uninitialized_variables/VarIsInitializedOp_256" input: "report_uninitialized_variables/VarIsInitializedOp_257" input: "report_uninitialized_variables/VarIsInitializedOp_258" input: "report_uninitialized_variables/VarIsInitializedOp_259" input: "report_uninitialized_variables/VarIsInitializedOp_260" input: "report_uninitialized_variables/VarIsInitializedOp_261" input: "report_uninitialized_variables/VarIsInitializedOp_262" input: "report_uninitialized_variables/VarIsInitializedOp_263" input: "report_uninitialized_variables/VarIsInitializedOp_264" input: "report_uninitialized_variables/VarIsInitializedOp_265" input: "report_uninitialized_variables/VarIsInitializedOp_266" input: "report_uninitialized_variables/VarIsInitializedOp_267" input: "report_uninitialized_variables/VarIsInitializedOp_268" input: "report_uninitialized_variables/VarIsInitializedOp_269" input: "report_uninitialized_variables/VarIsInitializedOp_270" input: "report_uninitialized_variables/VarIsInitializedOp_271" input: "report_uninitialized_variables/VarIsInitializedOp_272" input: "report_uninitialized_variables/VarIsInitializedOp_273" input: "report_uninitialized_variables/VarIsInitializedOp_274" input: "report_uninitialized_variables/VarIsInitializedOp_275" input: "report_uninitialized_variables/VarIsInitializedOp_276" input: "report_uninitialized_variables/VarIsInitializedOp_277" input: "report_uninitialized_variables/VarIsInitializedOp_278" input: "report_uninitialized_variables/VarIsInitializedOp_279" input: "report_uninitialized_variables/VarIsInitializedOp_280" input: "report_uninitialized_variables/VarIsInitializedOp_281" input: "report_uninitialized_variables/VarIsInitializedOp_282" input: "report_uninitialized_variables/VarIsInitializedOp_283" input: "report_uninitialized_variables/VarIsInitializedOp_284" input: "report_uninitialized_variables/VarIsInitializedOp_285" input: "report_uninitialized_variables/VarIsInitializedOp_286" input: "report_uninitialized_variables/VarIsInitializedOp_287" input: "report_uninitialized_variables/VarIsInitializedOp_288" input: "report_uninitialized_variables/VarIsInitializedOp_289" input: "report_uninitialized_variables/VarIsInitializedOp_290" input: "report_uninitialized_variables/VarIsInitializedOp_291" input: "report_uninitialized_variables/VarIsInitializedOp_292" input: "report_uninitialized_variables/VarIsInitializedOp_293" input: "report_uninitialized_variables/VarIsInitializedOp_294" input: "report_uninitialized_variables/VarIsInitializedOp_295" input: "report_uninitialized_variables/VarIsInitializedOp_296" input: "report_uninitialized_variables/VarIsInitializedOp_297" input: "report_uninitialized_variables/VarIsInitializedOp_298" input: "report_uninitialized_variables/VarIsInitializedOp_299" input: "report_uninitialized_variables/VarIsInitializedOp_300" input: "report_uninitialized_variables/VarIsInitializedOp_301" input: "report_uninitialized_variables/VarIsInitializedOp_302" input: "report_uninitialized_variables/VarIsInitializedOp_303" input: "report_uninitialized_variables/VarIsInitializedOp_304" input: "report_uninitialized_variables/VarIsInitializedOp_305" input: "report_uninitialized_variables/VarIsInitializedOp_306" input: "report_uninitialized_variables/VarIsInitializedOp_307" input: "report_uninitialized_variables/VarIsInitializedOp_308" input: "report_uninitialized_variables/VarIsInitializedOp_309" input: "report_uninitialized_variables/VarIsInitializedOp_310" input: "report_uninitialized_variables/VarIsInitializedOp_311" input: "report_uninitialized_variables/VarIsInitializedOp_312" input: "report_uninitialized_variables/VarIsInitializedOp_313" input: "report_uninitialized_variables/VarIsInitializedOp_314" input: "report_uninitialized_variables/VarIsInitializedOp_315" input: "report_uninitialized_variables/VarIsInitializedOp_316" input: "report_uninitialized_variables/VarIsInitializedOp_317" input: "report_uninitialized_variables/VarIsInitializedOp_318" input: "report_uninitialized_variables/VarIsInitializedOp_319" input: "report_uninitialized_variables/VarIsInitializedOp_320" input: "report_uninitialized_variables/VarIsInitializedOp_321" input: "report_uninitialized_variables/VarIsInitializedOp_322" input: "report_uninitialized_variables/VarIsInitializedOp_323" input: "report_uninitialized_variables/VarIsInitializedOp_324" input: "report_uninitialized_variables/VarIsInitializedOp_325" input: "report_uninitialized_variables/VarIsInitializedOp_326" input: "report_uninitialized_variables/VarIsInitializedOp_327" input: "report_uninitialized_variables/VarIsInitializedOp_328" input: "report_uninitialized_variables/VarIsInitializedOp_329" input: "report_uninitialized_variables/VarIsInitializedOp_330" input: "report_uninitialized_variables/VarIsInitializedOp_331" input: "report_uninitialized_variables/VarIsInitializedOp_332" input: "report_uninitialized_variables/VarIsInitializedOp_333" input: "report_uninitialized_variables/VarIsInitializedOp_334" input: "report_uninitialized_variables/VarIsInitializedOp_335" input: "report_uninitialized_variables/VarIsInitializedOp_336" input: "report_uninitialized_variables/VarIsInitializedOp_337" input: "report_uninitialized_variables/VarIsInitializedOp_338" input: "report_uninitialized_variables/VarIsInitializedOp_339" input: "report_uninitialized_variables/VarIsInitializedOp_340" input: "report_uninitialized_variables/VarIsInitializedOp_341" input: "report_uninitialized_variables/VarIsInitializedOp_342" input: "report_uninitialized_variables/VarIsInitializedOp_343" input: "report_uninitialized_variables/VarIsInitializedOp_344" input: "report_uninitialized_variables/VarIsInitializedOp_345" input: "report_uninitialized_variables/VarIsInitializedOp_346" input: "report_uninitialized_variables/VarIsInitializedOp_347" input: "report_uninitialized_variables/VarIsInitializedOp_348" input: "report_uninitialized_variables/VarIsInitializedOp_349" input: "report_uninitialized_variables/VarIsInitializedOp_350" input: "report_uninitialized_variables/VarIsInitializedOp_351" input: "report_uninitialized_variables/VarIsInitializedOp_352" input: "report_uninitialized_variables/VarIsInitializedOp_353" input: "report_uninitialized_variables/VarIsInitializedOp_354" input: "report_uninitialized_variables/VarIsInitializedOp_355" input: "report_uninitialized_variables/VarIsInitializedOp_356" input: "report_uninitialized_variables/VarIsInitializedOp_357" input: "report_uninitialized_variables/VarIsInitializedOp_358" input: "report_uninitialized_variables/VarIsInitializedOp_359" input: "report_uninitialized_variables/VarIsInitializedOp_360" input: "report_uninitialized_variables/VarIsInitializedOp_361" input: "report_uninitialized_variables/VarIsInitializedOp_362" input: "report_uninitialized_variables/VarIsInitializedOp_363" input: "report_uninitialized_variables/VarIsInitializedOp_364" input: "report_uninitialized_variables/VarIsInitializedOp_365" input: "report_uninitialized_variables/VarIsInitializedOp_366" input: "report_uninitialized_variables/VarIsInitializedOp_367" input: "report_uninitialized_variables/VarIsInitializedOp_368" input: "report_uninitialized_variables/VarIsInitializedOp_369" input: "report_uninitialized_variables/VarIsInitializedOp_370" input: "report_uninitialized_variables/VarIsInitializedOp_371" input: "report_uninitialized_variables/VarIsInitializedOp_372" input: "report_uninitialized_variables/VarIsInitializedOp_373" input: "report_uninitialized_variables/VarIsInitializedOp_374" input: "report_uninitialized_variables/VarIsInitializedOp_375" input: "report_uninitialized_variables/VarIsInitializedOp_376" input: "report_uninitialized_variables/VarIsInitializedOp_377" input: "report_uninitialized_variables/VarIsInitializedOp_378" input: "report_uninitialized_variables/VarIsInitializedOp_379" input: "report_uninitialized_variables/VarIsInitializedOp_380" input: "report_uninitialized_variables/VarIsInitializedOp_381" input: "report_uninitialized_variables/VarIsInitializedOp_382" input: "report_uninitialized_variables/VarIsInitializedOp_383" input: "report_uninitialized_variables/VarIsInitializedOp_384" input: "report_uninitialized_variables/VarIsInitializedOp_385" input: "report_uninitialized_variables/VarIsInitializedOp_386" input: "report_uninitialized_variables/VarIsInitializedOp_387" input: "report_uninitialized_variables/VarIsInitializedOp_388" input: "report_uninitialized_variables/VarIsInitializedOp_389" input: "report_uninitialized_variables/VarIsInitializedOp_390" input: "report_uninitialized_variables/VarIsInitializedOp_391" input: "report_uninitialized_variables/VarIsInitializedOp_392" input: "report_uninitialized_variables/VarIsInitializedOp_393" input: "report_uninitialized_variables/VarIsInitializedOp_394" input: "report_uninitialized_variables/VarIsInitializedOp_395" input: "report_uninitialized_variables/VarIsInitializedOp_396" input: "report_uninitialized_variables/VarIsInitializedOp_397" input: "report_uninitialized_variables/VarIsInitializedOp_398" input: "report_uninitialized_variables/VarIsInitializedOp_399" input: "report_uninitialized_variables/VarIsInitializedOp_400" input: "report_uninitialized_variables/VarIsInitializedOp_401" input: "report_uninitialized_variables/VarIsInitializedOp_402" input: "report_uninitialized_variables/VarIsInitializedOp_403" input: "report_uninitialized_variables/VarIsInitializedOp_404" input: "report_uninitialized_variables/VarIsInitializedOp_405" input: "report_uninitialized_variables/VarIsInitializedOp_406" input: "report_uninitialized_variables/VarIsInitializedOp_407" input: "report_uninitialized_variables/VarIsInitializedOp_408" input: "report_uninitialized_variables/VarIsInitializedOp_409" input: "report_uninitialized_variables/VarIsInitializedOp_410" input: "report_uninitialized_variables/VarIsInitializedOp_411" input: "report_uninitialized_variables/VarIsInitializedOp_412" input: "report_uninitialized_variables/VarIsInitializedOp_413" input: "report_uninitialized_variables/VarIsInitializedOp_414" input: "report_uninitialized_variables/VarIsInitializedOp_415" input: "report_uninitialized_variables/VarIsInitializedOp_416" input: "report_uninitialized_variables/VarIsInitializedOp_417" input: "report_uninitialized_variables/VarIsInitializedOp_418" input: "report_uninitialized_variables/VarIsInitializedOp_419" input: "report_uninitialized_variables/VarIsInitializedOp_420" input: "report_uninitialized_variables/VarIsInitializedOp_421" input: "report_uninitialized_variables/VarIsInitializedOp_422" input: "report_uninitialized_variables/VarIsInitializedOp_423" input: "report_uninitialized_variables/VarIsInitializedOp_424" input: "report_uninitialized_variables/VarIsInitializedOp_425" input: "report_uninitialized_variables/VarIsInitializedOp_426" input: "report_uninitialized_variables/VarIsInitializedOp_427" input: "report_uninitialized_variables/VarIsInitializedOp_428" input: "report_uninitialized_variables/VarIsInitializedOp_429" input: "report_uninitialized_variables/VarIsInitializedOp_430" input: "report_uninitialized_variables/VarIsInitializedOp_431" input: "report_uninitialized_variables/VarIsInitializedOp_432" input: "report_uninitialized_variables/VarIsInitializedOp_433" input: "report_uninitialized_variables/VarIsInitializedOp_434" input: "report_uninitialized_variables/VarIsInitializedOp_435" input: "report_uninitialized_variables/VarIsInitializedOp_436" input: "report_uninitialized_variables/VarIsInitializedOp_437" input: "report_uninitialized_variables/VarIsInitializedOp_438" input: "report_uninitialized_variables/VarIsInitializedOp_439" input: "report_uninitialized_variables/VarIsInitializedOp_440" input: "report_uninitialized_variables/VarIsInitializedOp_441" input: "report_uninitialized_variables/VarIsInitializedOp_442" input: "report_uninitialized_variables/VarIsInitializedOp_443" input: "report_uninitialized_variables/VarIsInitializedOp_444" input: "report_uninitialized_variables/VarIsInitializedOp_445" input: "report_uninitialized_variables/VarIsInitializedOp_446" input: "report_uninitialized_variables/VarIsInitializedOp_447" input: "report_uninitialized_variables/VarIsInitializedOp_448" input: "report_uninitialized_variables/VarIsInitializedOp_449" input: "report_uninitialized_variables/VarIsInitializedOp_450" input: "report_uninitialized_variables/VarIsInitializedOp_451" input: "report_uninitialized_variables/VarIsInitializedOp_452" input: "report_uninitialized_variables/VarIsInitializedOp_453" input: "report_uninitialized_variables/VarIsInitializedOp_454" input: "report_uninitialized_variables/VarIsInitializedOp_455" input: "report_uninitialized_variables/VarIsInitializedOp_456" input: "report_uninitialized_variables/VarIsInitializedOp_457" input: "report_uninitialized_variables/VarIsInitializedOp_458" input: "report_uninitialized_variables/VarIsInitializedOp_459" input: "report_uninitialized_variables/VarIsInitializedOp_460" input: "report_uninitialized_variables/VarIsInitializedOp_461" input: "report_uninitialized_variables/VarIsInitializedOp_462" input: "report_uninitialized_variables/VarIsInitializedOp_463" input: "report_uninitialized_variables/VarIsInitializedOp_464" input: "report_uninitialized_variables/VarIsInitializedOp_465" input: "report_uninitialized_variables/VarIsInitializedOp_466" input: "report_uninitialized_variables/VarIsInitializedOp_467" input: "report_uninitialized_variables/VarIsInitializedOp_468" input: "report_uninitialized_variables/VarIsInitializedOp_469" input: "report_uninitialized_variables/VarIsInitializedOp_470" input: "report_uninitialized_variables/VarIsInitializedOp_471" input: "report_uninitialized_variables/VarIsInitializedOp_472" input: "report_uninitialized_variables/VarIsInitializedOp_473" input: "report_uninitialized_variables/VarIsInitializedOp_474" input: "report_uninitialized_variables/VarIsInitializedOp_475" input: "report_uninitialized_variables/VarIsInitializedOp_476" input: "report_uninitialized_variables/VarIsInitializedOp_477" input: "report_uninitialized_variables/VarIsInitializedOp_478" input: "report_uninitialized_variables/VarIsInitializedOp_479" input: "report_uninitialized_variables/VarIsInitializedOp_480" input: "report_uninitialized_variables/VarIsInitializedOp_481" input: "report_uninitialized_variables/VarIsInitializedOp_482" input: "report_uninitialized_variables/VarIsInitializedOp_483" input: "report_uninitialized_variables/VarIsInitializedOp_484" input: "report_uninitialized_variables/VarIsInitializedOp_485" input: "report_uninitialized_variables/VarIsInitializedOp_486" input: "report_uninitialized_variables/VarIsInitializedOp_487" input: "report_uninitialized_variables/VarIsInitializedOp_488" input: "report_uninitialized_variables/VarIsInitializedOp_489" input: "report_uninitialized_variables/VarIsInitializedOp_490" input: "report_uninitialized_variables/VarIsInitializedOp_491" input: "report_uninitialized_variables/VarIsInitializedOp_492" input: "report_uninitialized_variables/VarIsInitializedOp_493" input: "report_uninitialized_variables/VarIsInitializedOp_494" input: "report_uninitialized_variables/VarIsInitializedOp_495" input: "report_uninitialized_variables/VarIsInitializedOp_496" input: "report_uninitialized_variables/VarIsInitializedOp_497" input: "report_uninitialized_variables/VarIsInitializedOp_498" input: "report_uninitialized_variables/VarIsInitializedOp_499" input: "report_uninitialized_variables/VarIsInitializedOp_500" input: "report_uninitialized_variables/VarIsInitializedOp_501" input: "report_uninitialized_variables/VarIsInitializedOp_502" input: "report_uninitialized_variables/VarIsInitializedOp_503" input: "report_uninitialized_variables/VarIsInitializedOp_504" input: "report_uninitialized_variables/VarIsInitializedOp_505" input: "report_uninitialized_variables/VarIsInitializedOp_506" input: "report_uninitialized_variables/VarIsInitializedOp_507" input: "report_uninitialized_variables/VarIsInitializedOp_508" input: "report_uninitialized_variables/VarIsInitializedOp_509" input: "report_uninitialized_variables/VarIsInitializedOp_510" input: "report_uninitialized_variables/VarIsInitializedOp_511" input: "report_uninitialized_variables/VarIsInitializedOp_512" input: "report_uninitialized_variables/VarIsInitializedOp_513" input: "report_uninitialized_variables/VarIsInitializedOp_514" input: "report_uninitialized_variables/VarIsInitializedOp_515" input: "report_uninitialized_variables/VarIsInitializedOp_516" input: "report_uninitialized_variables/VarIsInitializedOp_517" input: "report_uninitialized_variables/VarIsInitializedOp_518" input: "report_uninitialized_variables/VarIsInitializedOp_519" input: "report_uninitialized_variables/VarIsInitializedOp_520" input: "report_uninitialized_variables/VarIsInitializedOp_521" input: "report_uninitialized_variables/VarIsInitializedOp_522" input: "report_uninitialized_variables/VarIsInitializedOp_523" input: "report_uninitialized_variables/VarIsInitializedOp_524" input: "report_uninitialized_variables/VarIsInitializedOp_525" input: "report_uninitialized_variables/VarIsInitializedOp_526" input: "report_uninitialized_variables/VarIsInitializedOp_527" input: "report_uninitialized_variables/VarIsInitializedOp_528" input: "report_uninitialized_variables/VarIsInitializedOp_529" input: "report_uninitialized_variables/VarIsInitializedOp_530" input: "report_uninitialized_variables/VarIsInitializedOp_531" input: "report_uninitialized_variables/VarIsInitializedOp_532" input: "report_uninitialized_variables/VarIsInitializedOp_533" input: "report_uninitialized_variables/VarIsInitializedOp_534" input: "report_uninitialized_variables/VarIsInitializedOp_535" input: "report_uninitialized_variables/VarIsInitializedOp_536" input: "report_uninitialized_variables/VarIsInitializedOp_537" input: "report_uninitialized_variables/VarIsInitializedOp_538" input: "report_uninitialized_variables/VarIsInitializedOp_539" input: "report_uninitialized_variables/VarIsInitializedOp_540" input: "report_uninitialized_variables/VarIsInitializedOp_541" input: "report_uninitialized_variables/VarIsInitializedOp_542" input: "report_uninitialized_variables/VarIsInitializedOp_543" input: "report_uninitialized_variables/VarIsInitializedOp_544" input: "report_uninitialized_variables/VarIsInitializedOp_545" input: "report_uninitialized_variables/VarIsInitializedOp_546" input: "report_uninitialized_variables/VarIsInitializedOp_547" input: "report_uninitialized_variables/VarIsInitializedOp_548" input: "report_uninitialized_variables/VarIsInitializedOp_549" input: "report_uninitialized_variables/VarIsInitializedOp_550" input: "report_uninitialized_variables/VarIsInitializedOp_551" input: "report_uninitialized_variables/VarIsInitializedOp_552" input: "report_uninitialized_variables/VarIsInitializedOp_553" input: "report_uninitialized_variables/VarIsInitializedOp_554" input: "report_uninitialized_variables/VarIsInitializedOp_555" input: "report_uninitialized_variables/VarIsInitializedOp_556" input: "report_uninitialized_variables/VarIsInitializedOp_557" input: "report_uninitialized_variables/VarIsInitializedOp_558" input: "report_uninitialized_variables/VarIsInitializedOp_559" input: "report_uninitialized_variables/VarIsInitializedOp_560" input: "report_uninitialized_variables/VarIsInitializedOp_561" input: "report_uninitialized_variables/VarIsInitializedOp_562" input: "report_uninitialized_variables/VarIsInitializedOp_563" input: "report_uninitialized_variables/VarIsInitializedOp_564" input: "report_uninitialized_variables/VarIsInitializedOp_565" input: "report_uninitialized_variables/VarIsInitializedOp_566" input: "report_uninitialized_variables/VarIsInitializedOp_567" input: "report_uninitialized_variables/VarIsInitializedOp_568" input: "report_uninitialized_variables/VarIsInitializedOp_569" input: "report_uninitialized_variables/VarIsInitializedOp_570" input: "report_uninitialized_variables/VarIsInitializedOp_571" input: "report_uninitialized_variables/VarIsInitializedOp_572" input: "report_uninitialized_variables/VarIsInitializedOp_573" input: "report_uninitialized_variables/VarIsInitializedOp_574" input: "report_uninitialized_variables/VarIsInitializedOp_575" input: "report_uninitialized_variables/VarIsInitializedOp_576" input: "report_uninitialized_variables/VarIsInitializedOp_577" input: "report_uninitialized_variables/VarIsInitializedOp_578" input: "report_uninitialized_variables/VarIsInitializedOp_579" input: "report_uninitialized_variables/VarIsInitializedOp_580" input: "report_uninitialized_variables/VarIsInitializedOp_581" input: "report_uninitialized_variables/VarIsInitializedOp_582" input: "report_uninitialized_variables/VarIsInitializedOp_583" input: "report_uninitialized_variables/VarIsInitializedOp_584" input: "report_uninitialized_variables/VarIsInitializedOp_585" input: "report_uninitialized_variables/VarIsInitializedOp_586" input: "report_uninitialized_variables/VarIsInitializedOp_587" input: "report_uninitialized_variables/VarIsInitializedOp_588" input: "report_uninitialized_variables/VarIsInitializedOp_589" input: "report_uninitialized_variables/VarIsInitializedOp_590" input: "report_uninitialized_variables/VarIsInitializedOp_591" input: "report_uninitialized_variables/VarIsInitializedOp_592" input: "report_uninitialized_variables/VarIsInitializedOp_593" input: "report_uninitialized_variables/VarIsInitializedOp_594" input: "report_uninitialized_variables/VarIsInitializedOp_595" input: "report_uninitialized_variables/VarIsInitializedOp_596" input: "report_uninitialized_variables/VarIsInitializedOp_597" input: "report_uninitialized_variables/VarIsInitializedOp_598" input: "report_uninitialized_variables/VarIsInitializedOp_599" input: "report_uninitialized_variables/VarIsInitializedOp_600" input: "report_uninitialized_variables/VarIsInitializedOp_601" input: "report_uninitialized_variables/VarIsInitializedOp_602" input: "report_uninitialized_variables/VarIsInitializedOp_603" device: "/device:CPU:0" attr { key: "N" value { i: 604 } } attr { key: "T" value { type: DT_BOOL } } attr { key: "_output_shapes" value { list { shape { dim { size: 604 } } } } } attr { key: "axis" value { i: 0 } } } node { name: "report_uninitialized_variables/LogicalNot" op: "LogicalNot" input: "report_uninitialized_variables/stack" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 604 } } } } } } node { name: "report_uninitialized_variables/Const" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 604 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 604 } } string_val: "global_step" string_val: "bert/embeddings/word_embeddings" string_val: "bert/embeddings/token_type_embeddings" string_val: "bert/embeddings/position_embeddings" string_val: "bert/embeddings/layer_normalization/gamma" string_val: "bert/embeddings/layer_normalization/beta" string_val: "bert/encoder/layer_0/attention/self/query/kernel" string_val: "bert/encoder/layer_0/attention/self/query/bias" string_val: "bert/encoder/layer_0/attention/self/key/kernel" string_val: "bert/encoder/layer_0/attention/self/key/bias" string_val: "bert/encoder/layer_0/attention/self/value/kernel" string_val: "bert/encoder/layer_0/attention/self/value/bias" string_val: "bert/encoder/layer_0/attention/output/dense/kernel" string_val: "bert/encoder/layer_0/attention/output/dense/bias" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta" string_val: "bert/encoder/layer_0/intermediate/dense/kernel" string_val: "bert/encoder/layer_0/intermediate/dense/bias" string_val: "bert/encoder/layer_0/output/dense/kernel" string_val: "bert/encoder/layer_0/output/dense/bias" string_val: "bert/encoder/layer_0/output/layer_normalization_2/gamma" string_val: "bert/encoder/layer_0/output/layer_normalization_2/beta" string_val: "bert/encoder/layer_1/attention/self/query/kernel" string_val: "bert/encoder/layer_1/attention/self/query/bias" string_val: "bert/encoder/layer_1/attention/self/key/kernel" string_val: "bert/encoder/layer_1/attention/self/key/bias" string_val: "bert/encoder/layer_1/attention/self/value/kernel" string_val: "bert/encoder/layer_1/attention/self/value/bias" string_val: "bert/encoder/layer_1/attention/output/dense/kernel" string_val: "bert/encoder/layer_1/attention/output/dense/bias" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta" string_val: "bert/encoder/layer_1/intermediate/dense/kernel" string_val: "bert/encoder/layer_1/intermediate/dense/bias" string_val: "bert/encoder/layer_1/output/dense/kernel" string_val: "bert/encoder/layer_1/output/dense/bias" string_val: "bert/encoder/layer_1/output/layer_normalization_4/gamma" string_val: "bert/encoder/layer_1/output/layer_normalization_4/beta" string_val: "bert/encoder/layer_2/attention/self/query/kernel" string_val: "bert/encoder/layer_2/attention/self/query/bias" string_val: "bert/encoder/layer_2/attention/self/key/kernel" string_val: "bert/encoder/layer_2/attention/self/key/bias" string_val: "bert/encoder/layer_2/attention/self/value/kernel" string_val: "bert/encoder/layer_2/attention/self/value/bias" string_val: "bert/encoder/layer_2/attention/output/dense/kernel" string_val: "bert/encoder/layer_2/attention/output/dense/bias" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta" string_val: "bert/encoder/layer_2/intermediate/dense/kernel" string_val: "bert/encoder/layer_2/intermediate/dense/bias" string_val: "bert/encoder/layer_2/output/dense/kernel" string_val: "bert/encoder/layer_2/output/dense/bias" string_val: "bert/encoder/layer_2/output/layer_normalization_6/gamma" string_val: "bert/encoder/layer_2/output/layer_normalization_6/beta" string_val: "bert/encoder/layer_3/attention/self/query/kernel" string_val: "bert/encoder/layer_3/attention/self/query/bias" string_val: "bert/encoder/layer_3/attention/self/key/kernel" string_val: "bert/encoder/layer_3/attention/self/key/bias" string_val: "bert/encoder/layer_3/attention/self/value/kernel" string_val: "bert/encoder/layer_3/attention/self/value/bias" string_val: "bert/encoder/layer_3/attention/output/dense/kernel" string_val: "bert/encoder/layer_3/attention/output/dense/bias" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta" string_val: "bert/encoder/layer_3/intermediate/dense/kernel" string_val: "bert/encoder/layer_3/intermediate/dense/bias" string_val: "bert/encoder/layer_3/output/dense/kernel" string_val: "bert/encoder/layer_3/output/dense/bias" string_val: "bert/encoder/layer_3/output/layer_normalization_8/gamma" string_val: "bert/encoder/layer_3/output/layer_normalization_8/beta" string_val: "bert/encoder/layer_4/attention/self/query/kernel" string_val: "bert/encoder/layer_4/attention/self/query/bias" string_val: "bert/encoder/layer_4/attention/self/key/kernel" string_val: "bert/encoder/layer_4/attention/self/key/bias" string_val: "bert/encoder/layer_4/attention/self/value/kernel" string_val: "bert/encoder/layer_4/attention/self/value/bias" string_val: "bert/encoder/layer_4/attention/output/dense/kernel" string_val: "bert/encoder/layer_4/attention/output/dense/bias" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta" string_val: "bert/encoder/layer_4/intermediate/dense/kernel" string_val: "bert/encoder/layer_4/intermediate/dense/bias" string_val: "bert/encoder/layer_4/output/dense/kernel" string_val: "bert/encoder/layer_4/output/dense/bias" string_val: "bert/encoder/layer_4/output/layer_normalization_10/gamma" string_val: "bert/encoder/layer_4/output/layer_normalization_10/beta" string_val: "bert/encoder/layer_5/attention/self/query/kernel" string_val: "bert/encoder/layer_5/attention/self/query/bias" string_val: "bert/encoder/layer_5/attention/self/key/kernel" string_val: "bert/encoder/layer_5/attention/self/key/bias" string_val: "bert/encoder/layer_5/attention/self/value/kernel" string_val: "bert/encoder/layer_5/attention/self/value/bias" string_val: "bert/encoder/layer_5/attention/output/dense/kernel" string_val: "bert/encoder/layer_5/attention/output/dense/bias" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta" string_val: "bert/encoder/layer_5/intermediate/dense/kernel" string_val: "bert/encoder/layer_5/intermediate/dense/bias" string_val: "bert/encoder/layer_5/output/dense/kernel" string_val: "bert/encoder/layer_5/output/dense/bias" string_val: "bert/encoder/layer_5/output/layer_normalization_12/gamma" string_val: "bert/encoder/layer_5/output/layer_normalization_12/beta" string_val: "bert/encoder/layer_6/attention/self/query/kernel" string_val: "bert/encoder/layer_6/attention/self/query/bias" string_val: "bert/encoder/layer_6/attention/self/key/kernel" string_val: "bert/encoder/layer_6/attention/self/key/bias" string_val: "bert/encoder/layer_6/attention/self/value/kernel" string_val: "bert/encoder/layer_6/attention/self/value/bias" string_val: "bert/encoder/layer_6/attention/output/dense/kernel" string_val: "bert/encoder/layer_6/attention/output/dense/bias" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta" string_val: "bert/encoder/layer_6/intermediate/dense/kernel" string_val: "bert/encoder/layer_6/intermediate/dense/bias" string_val: "bert/encoder/layer_6/output/dense/kernel" string_val: "bert/encoder/layer_6/output/dense/bias" string_val: "bert/encoder/layer_6/output/layer_normalization_14/gamma" string_val: "bert/encoder/layer_6/output/layer_normalization_14/beta" string_val: "bert/encoder/layer_7/attention/self/query/kernel" string_val: "bert/encoder/layer_7/attention/self/query/bias" string_val: "bert/encoder/layer_7/attention/self/key/kernel" string_val: "bert/encoder/layer_7/attention/self/key/bias" string_val: "bert/encoder/layer_7/attention/self/value/kernel" string_val: "bert/encoder/layer_7/attention/self/value/bias" string_val: "bert/encoder/layer_7/attention/output/dense/kernel" string_val: "bert/encoder/layer_7/attention/output/dense/bias" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta" string_val: "bert/encoder/layer_7/intermediate/dense/kernel" string_val: "bert/encoder/layer_7/intermediate/dense/bias" string_val: "bert/encoder/layer_7/output/dense/kernel" string_val: "bert/encoder/layer_7/output/dense/bias" string_val: "bert/encoder/layer_7/output/layer_normalization_16/gamma" string_val: "bert/encoder/layer_7/output/layer_normalization_16/beta" string_val: "bert/encoder/layer_8/attention/self/query/kernel" string_val: "bert/encoder/layer_8/attention/self/query/bias" string_val: "bert/encoder/layer_8/attention/self/key/kernel" string_val: "bert/encoder/layer_8/attention/self/key/bias" string_val: "bert/encoder/layer_8/attention/self/value/kernel" string_val: "bert/encoder/layer_8/attention/self/value/bias" string_val: "bert/encoder/layer_8/attention/output/dense/kernel" string_val: "bert/encoder/layer_8/attention/output/dense/bias" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta" string_val: "bert/encoder/layer_8/intermediate/dense/kernel" string_val: "bert/encoder/layer_8/intermediate/dense/bias" string_val: "bert/encoder/layer_8/output/dense/kernel" string_val: "bert/encoder/layer_8/output/dense/bias" string_val: "bert/encoder/layer_8/output/layer_normalization_18/gamma" string_val: "bert/encoder/layer_8/output/layer_normalization_18/beta" string_val: "bert/encoder/layer_9/attention/self/query/kernel" string_val: "bert/encoder/layer_9/attention/self/query/bias" string_val: "bert/encoder/layer_9/attention/self/key/kernel" string_val: "bert/encoder/layer_9/attention/self/key/bias" string_val: "bert/encoder/layer_9/attention/self/value/kernel" string_val: "bert/encoder/layer_9/attention/self/value/bias" string_val: "bert/encoder/layer_9/attention/output/dense/kernel" string_val: "bert/encoder/layer_9/attention/output/dense/bias" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta" string_val: "bert/encoder/layer_9/intermediate/dense/kernel" string_val: "bert/encoder/layer_9/intermediate/dense/bias" string_val: "bert/encoder/layer_9/output/dense/kernel" string_val: "bert/encoder/layer_9/output/dense/bias" string_val: "bert/encoder/layer_9/output/layer_normalization_20/gamma" string_val: "bert/encoder/layer_9/output/layer_normalization_20/beta" string_val: "bert/encoder/layer_10/attention/self/query/kernel" string_val: "bert/encoder/layer_10/attention/self/query/bias" string_val: "bert/encoder/layer_10/attention/self/key/kernel" string_val: "bert/encoder/layer_10/attention/self/key/bias" string_val: "bert/encoder/layer_10/attention/self/value/kernel" string_val: "bert/encoder/layer_10/attention/self/value/bias" string_val: "bert/encoder/layer_10/attention/output/dense/kernel" string_val: "bert/encoder/layer_10/attention/output/dense/bias" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta" string_val: "bert/encoder/layer_10/intermediate/dense/kernel" string_val: "bert/encoder/layer_10/intermediate/dense/bias" string_val: "bert/encoder/layer_10/output/dense/kernel" string_val: "bert/encoder/layer_10/output/dense/bias" string_val: "bert/encoder/layer_10/output/layer_normalization_22/gamma" string_val: "bert/encoder/layer_10/output/layer_normalization_22/beta" string_val: "bert/encoder/layer_11/attention/self/query/kernel" string_val: "bert/encoder/layer_11/attention/self/query/bias" string_val: "bert/encoder/layer_11/attention/self/key/kernel" string_val: "bert/encoder/layer_11/attention/self/key/bias" string_val: "bert/encoder/layer_11/attention/self/value/kernel" string_val: "bert/encoder/layer_11/attention/self/value/bias" string_val: "bert/encoder/layer_11/attention/output/dense/kernel" string_val: "bert/encoder/layer_11/attention/output/dense/bias" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta" string_val: "bert/encoder/layer_11/intermediate/dense/kernel" string_val: "bert/encoder/layer_11/intermediate/dense/bias" string_val: "bert/encoder/layer_11/output/dense/kernel" string_val: "bert/encoder/layer_11/output/dense/bias" string_val: "bert/encoder/layer_11/output/layer_normalization_24/gamma" string_val: "bert/encoder/layer_11/output/layer_normalization_24/beta" string_val: "bert/pooler/dense/kernel" string_val: "bert/pooler/dense/bias" string_val: "output_weights" string_val: "output_bias" string_val: "bert/embeddings/word_embeddings/adam_m" string_val: "bert/embeddings/word_embeddings/adam_v" string_val: "bert/embeddings/token_type_embeddings/adam_m" string_val: "bert/embeddings/token_type_embeddings/adam_v" string_val: "bert/embeddings/position_embeddings/adam_m" string_val: "bert/embeddings/position_embeddings/adam_v" string_val: "bert/embeddings/layer_normalization/gamma/adam_m" string_val: "bert/embeddings/layer_normalization/gamma/adam_v" string_val: "bert/embeddings/layer_normalization/beta/adam_m" string_val: "bert/embeddings/layer_normalization/beta/adam_v" string_val: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" string_val: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_0/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/output/dense/bias/adam_m" string_val: "bert/encoder/layer_0/output/dense/bias/adam_v" string_val: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" string_val: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" string_val: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" string_val: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" string_val: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" string_val: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_1/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/output/dense/bias/adam_m" string_val: "bert/encoder/layer_1/output/dense/bias/adam_v" string_val: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" string_val: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" string_val: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" string_val: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" string_val: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" string_val: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_2/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/output/dense/bias/adam_m" string_val: "bert/encoder/layer_2/output/dense/bias/adam_v" string_val: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" string_val: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" string_val: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" string_val: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" string_val: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" string_val: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_3/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/output/dense/bias/adam_m" string_val: "bert/encoder/layer_3/output/dense/bias/adam_v" string_val: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" string_val: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" string_val: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" string_val: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" string_val: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" string_val: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_4/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/output/dense/bias/adam_m" string_val: "bert/encoder/layer_4/output/dense/bias/adam_v" string_val: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" string_val: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" string_val: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" string_val: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" string_val: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" string_val: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_5/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/output/dense/bias/adam_m" string_val: "bert/encoder/layer_5/output/dense/bias/adam_v" string_val: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" string_val: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" string_val: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" string_val: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" string_val: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" string_val: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_6/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/output/dense/bias/adam_m" string_val: "bert/encoder/layer_6/output/dense/bias/adam_v" string_val: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" string_val: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" string_val: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" string_val: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" string_val: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" string_val: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_7/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/output/dense/bias/adam_m" string_val: "bert/encoder/layer_7/output/dense/bias/adam_v" string_val: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" string_val: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" string_val: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" string_val: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" string_val: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" string_val: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_8/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/output/dense/bias/adam_m" string_val: "bert/encoder/layer_8/output/dense/bias/adam_v" string_val: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" string_val: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" string_val: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" string_val: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" string_val: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" string_val: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_9/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/output/dense/bias/adam_m" string_val: "bert/encoder/layer_9/output/dense/bias/adam_v" string_val: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" string_val: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" string_val: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" string_val: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" string_val: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" string_val: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_10/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/output/dense/bias/adam_m" string_val: "bert/encoder/layer_10/output/dense/bias/adam_v" string_val: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" string_val: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" string_val: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" string_val: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" string_val: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" string_val: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_11/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/output/dense/bias/adam_m" string_val: "bert/encoder/layer_11/output/dense/bias/adam_v" string_val: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" string_val: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" string_val: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" string_val: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" string_val: "bert/pooler/dense/kernel/adam_m" string_val: "bert/pooler/dense/kernel/adam_v" string_val: "bert/pooler/dense/bias/adam_m" string_val: "bert/pooler/dense/bias/adam_v" string_val: "output_weights/adam_m" string_val: "output_weights/adam_v" string_val: "output_bias/adam_m" string_val: "output_bias/adam_v" } } } } node { name: "report_uninitialized_variables/boolean_mask/Shape" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 604 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice/stack" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice/stack_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice/stack_2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice" op: "StridedSlice" input: "report_uninitialized_variables/boolean_mask/Shape" input: "report_uninitialized_variables/boolean_mask/strided_slice/stack" input: "report_uninitialized_variables/boolean_mask/strided_slice/stack_1" input: "report_uninitialized_variables/boolean_mask/strided_slice/stack_2" device: "/device:CPU:0" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 0 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "report_uninitialized_variables/boolean_mask/Prod/reduction_indices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables/boolean_mask/Prod" op: "Prod" input: "report_uninitialized_variables/boolean_mask/strided_slice" input: "report_uninitialized_variables/boolean_mask/Prod/reduction_indices" device: "/device:CPU:0" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "report_uninitialized_variables/boolean_mask/Shape_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 604 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack_2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice_1" op: "StridedSlice" input: "report_uninitialized_variables/boolean_mask/Shape_1" input: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack" input: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack_1" input: "report_uninitialized_variables/boolean_mask/strided_slice_1/stack_2" device: "/device:CPU:0" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "begin_mask" value { i: 1 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 0 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "report_uninitialized_variables/boolean_mask/Shape_2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 604 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack_2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables/boolean_mask/strided_slice_2" op: "StridedSlice" input: "report_uninitialized_variables/boolean_mask/Shape_2" input: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack" input: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack_1" input: "report_uninitialized_variables/boolean_mask/strided_slice_2/stack_2" device: "/device:CPU:0" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 1 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "report_uninitialized_variables/boolean_mask/concat/values_1" op: "Pack" input: "report_uninitialized_variables/boolean_mask/Prod" device: "/device:CPU:0" attr { key: "N" value { i: 1 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "axis" value { i: 0 } } } node { name: "report_uninitialized_variables/boolean_mask/concat/axis" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "report_uninitialized_variables/boolean_mask/concat" op: "ConcatV2" input: "report_uninitialized_variables/boolean_mask/strided_slice_1" input: "report_uninitialized_variables/boolean_mask/concat/values_1" input: "report_uninitialized_variables/boolean_mask/strided_slice_2" input: "report_uninitialized_variables/boolean_mask/concat/axis" device: "/device:CPU:0" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "report_uninitialized_variables/boolean_mask/Reshape" op: "Reshape" input: "report_uninitialized_variables/Const" input: "report_uninitialized_variables/boolean_mask/concat" device: "/device:CPU:0" attr { key: "T" value { type: DT_STRING } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 604 } } } } } } node { name: "report_uninitialized_variables/boolean_mask/Reshape_1/shape" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: -1 } } } } node { name: "report_uninitialized_variables/boolean_mask/Reshape_1" op: "Reshape" input: "report_uninitialized_variables/LogicalNot" input: "report_uninitialized_variables/boolean_mask/Reshape_1/shape" device: "/device:CPU:0" attr { key: "T" value { type: DT_BOOL } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 604 } } } } } } node { name: "report_uninitialized_variables/boolean_mask/Where" op: "Where" input: "report_uninitialized_variables/boolean_mask/Reshape_1" device: "/device:CPU:0" attr { key: "T" value { type: DT_BOOL } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } dim { size: 1 } } } } } } node { name: "report_uninitialized_variables/boolean_mask/Squeeze" op: "Squeeze" input: "report_uninitialized_variables/boolean_mask/Where" device: "/device:CPU:0" attr { key: "T" value { type: DT_INT64 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } } } } attr { key: "squeeze_dims" value { list { i: 1 } } } } node { name: "report_uninitialized_variables/boolean_mask/GatherV2/axis" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "report_uninitialized_variables/boolean_mask/GatherV2" op: "GatherV2" input: "report_uninitialized_variables/boolean_mask/Reshape" input: "report_uninitialized_variables/boolean_mask/Squeeze" input: "report_uninitialized_variables/boolean_mask/GatherV2/axis" device: "/device:CPU:0" attr { key: "Taxis" value { type: DT_INT32 } } attr { key: "Tindices" value { type: DT_INT64 } } attr { key: "Tparams" value { type: DT_STRING } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } } } } attr { key: "batch_dims" value { i: 0 } } } node { name: "report_uninitialized_resources/Const" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { } } } } } } node { name: "concat/axis" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "concat" op: "ConcatV2" input: "report_uninitialized_variables/boolean_mask/GatherV2" input: "report_uninitialized_resources/Const" input: "concat/axis" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_STRING } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp" op: "VarIsInitializedOp" input: "global_step" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_1" op: "VarIsInitializedOp" input: "bert/embeddings/word_embeddings" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_2" op: "VarIsInitializedOp" input: "bert/embeddings/token_type_embeddings" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_3" op: "VarIsInitializedOp" input: "bert/embeddings/position_embeddings" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_4" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_5" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_6" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_7" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_8" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_9" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_10" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_11" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_12" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_13" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_14" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_15" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_16" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_17" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_18" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_19" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_20" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_21" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_22" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_23" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_24" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_25" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_26" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_27" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_28" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_29" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_30" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_31" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_32" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_33" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_34" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_35" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_36" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_37" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_38" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_39" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_40" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_41" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_42" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_43" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_44" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_45" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_46" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_47" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_48" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_49" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_50" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_51" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_52" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_53" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_54" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_55" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_56" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_57" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_58" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_59" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_60" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_61" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_62" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_63" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_64" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_65" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_66" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_67" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_68" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_69" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_70" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_71" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_72" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_73" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_74" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_75" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_76" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_77" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_78" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_79" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_80" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_81" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_82" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_83" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_84" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_85" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_86" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_87" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_88" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_89" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_90" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_91" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_92" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_93" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_94" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_95" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_96" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_97" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_98" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_99" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_100" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_101" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_102" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_103" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_104" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_105" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_106" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_107" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_108" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_109" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_110" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_111" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_112" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_113" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_114" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_115" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_116" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_117" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_118" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_119" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_120" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_121" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_122" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_123" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_124" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_125" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_126" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_127" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_128" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_129" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_130" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_131" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_132" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_133" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_134" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_135" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_136" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_137" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_138" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_139" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_140" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_141" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_142" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_143" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_144" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_145" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_146" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_147" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_148" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_149" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_150" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_151" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_152" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_153" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_154" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_155" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_156" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_157" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_158" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_159" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_160" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_161" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_162" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_163" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_164" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_165" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_166" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_167" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_168" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_169" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_170" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_171" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_172" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_173" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_174" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_175" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_176" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_177" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_178" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_179" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_180" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_181" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_182" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_183" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_184" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_185" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_186" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_187" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_188" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_189" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_190" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_191" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_192" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_193" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_194" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_195" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_196" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_197" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_198" op: "VarIsInitializedOp" input: "bert/pooler/dense/kernel" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_199" op: "VarIsInitializedOp" input: "bert/pooler/dense/bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_200" op: "VarIsInitializedOp" input: "output_weights" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_201" op: "VarIsInitializedOp" input: "output_bias" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_202" op: "VarIsInitializedOp" input: "bert/embeddings/word_embeddings/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_203" op: "VarIsInitializedOp" input: "bert/embeddings/word_embeddings/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_204" op: "VarIsInitializedOp" input: "bert/embeddings/token_type_embeddings/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_205" op: "VarIsInitializedOp" input: "bert/embeddings/token_type_embeddings/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_206" op: "VarIsInitializedOp" input: "bert/embeddings/position_embeddings/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_207" op: "VarIsInitializedOp" input: "bert/embeddings/position_embeddings/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_208" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_209" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_210" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_211" op: "VarIsInitializedOp" input: "bert/embeddings/layer_normalization/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_212" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_213" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_214" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_215" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_216" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_217" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_218" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_219" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_220" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_221" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_222" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_223" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_224" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_225" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_226" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_227" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_228" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_229" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_230" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_231" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_232" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_233" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_234" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_235" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_236" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_237" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_238" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_239" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_240" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_241" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_242" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_243" op: "VarIsInitializedOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_244" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_245" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_246" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_247" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_248" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_249" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_250" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_251" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_252" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_253" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_254" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_255" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_256" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_257" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_258" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_259" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_260" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_261" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_262" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_263" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_264" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_265" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_266" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_267" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_268" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_269" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_270" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_271" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_272" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_273" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_274" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_275" op: "VarIsInitializedOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_276" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_277" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_278" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_279" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_280" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_281" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_282" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_283" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_284" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_285" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_286" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_287" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_288" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_289" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_290" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_291" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_292" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_293" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_294" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_295" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_296" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_297" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_298" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_299" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_300" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_301" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_302" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_303" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_304" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_305" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_306" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_307" op: "VarIsInitializedOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_308" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_309" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_310" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_311" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_312" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_313" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_314" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_315" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_316" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_317" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_318" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_319" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_320" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_321" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_322" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_323" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_324" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_325" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_326" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_327" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_328" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_329" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_330" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_331" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_332" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_333" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_334" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_335" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_336" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_337" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_338" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_339" op: "VarIsInitializedOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_340" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_341" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_342" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_343" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_344" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_345" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_346" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_347" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_348" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_349" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_350" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_351" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_352" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_353" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_354" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_355" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_356" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_357" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_358" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_359" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_360" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_361" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_362" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_363" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_364" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_365" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_366" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_367" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_368" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_369" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_370" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_371" op: "VarIsInitializedOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_372" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_373" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_374" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_375" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_376" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_377" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_378" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_379" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_380" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_381" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_382" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_383" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_384" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_385" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_386" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_387" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_388" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_389" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_390" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_391" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_392" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_393" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_394" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_395" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_396" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_397" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_398" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_399" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_400" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_401" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_402" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_403" op: "VarIsInitializedOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_404" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_405" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_406" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_407" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_408" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_409" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_410" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_411" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_412" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_413" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_414" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_415" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_416" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_417" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_418" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_419" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_420" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_421" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_422" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_423" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_424" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_425" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_426" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_427" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_428" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_429" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_430" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_431" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_432" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_433" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_434" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_435" op: "VarIsInitializedOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_436" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_437" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_438" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_439" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_440" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_441" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_442" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_443" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_444" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_445" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_446" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_447" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_448" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_449" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_450" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_451" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_452" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_453" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_454" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_455" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_456" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_457" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_458" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_459" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_460" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_461" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_462" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_463" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_464" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_465" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_466" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_467" op: "VarIsInitializedOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_468" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_469" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_470" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_471" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_472" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_473" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_474" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_475" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_476" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_477" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_478" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_479" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_480" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_481" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_482" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_483" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_484" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_485" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_486" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_487" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_488" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_489" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_490" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_491" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_492" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_493" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_494" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_495" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_496" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_497" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_498" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_499" op: "VarIsInitializedOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_500" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_501" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_502" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_503" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_504" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_505" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_506" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_507" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_508" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_509" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_510" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_511" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_512" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_513" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_514" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_515" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_516" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_517" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_518" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_519" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_520" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_521" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_522" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_523" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_524" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_525" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_526" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_527" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_528" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_529" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_530" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_531" op: "VarIsInitializedOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_532" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_533" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_534" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_535" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_536" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_537" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_538" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_539" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_540" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_541" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_542" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_543" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_544" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_545" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_546" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_547" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_548" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_549" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_550" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_551" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_552" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_553" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_554" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_555" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_556" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_557" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_558" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_559" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_560" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_561" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_562" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_563" op: "VarIsInitializedOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_564" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_565" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_566" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_567" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_568" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_569" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_570" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_571" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_572" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_573" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_574" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_575" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_576" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_577" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_578" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_579" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_580" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_581" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_582" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_583" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_584" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_585" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_586" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_587" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_588" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_589" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_590" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_591" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_592" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_593" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_594" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_595" op: "VarIsInitializedOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_596" op: "VarIsInitializedOp" input: "bert/pooler/dense/kernel/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_597" op: "VarIsInitializedOp" input: "bert/pooler/dense/kernel/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_598" op: "VarIsInitializedOp" input: "bert/pooler/dense/bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_599" op: "VarIsInitializedOp" input: "bert/pooler/dense/bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_600" op: "VarIsInitializedOp" input: "output_weights/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_601" op: "VarIsInitializedOp" input: "output_weights/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_602" op: "VarIsInitializedOp" input: "output_bias/adam_m" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/VarIsInitializedOp_603" op: "VarIsInitializedOp" input: "output_bias/adam_v" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "report_uninitialized_variables_1/stack" op: "Pack" input: "report_uninitialized_variables_1/VarIsInitializedOp" input: "report_uninitialized_variables_1/VarIsInitializedOp_1" input: "report_uninitialized_variables_1/VarIsInitializedOp_2" input: "report_uninitialized_variables_1/VarIsInitializedOp_3" input: "report_uninitialized_variables_1/VarIsInitializedOp_4" input: "report_uninitialized_variables_1/VarIsInitializedOp_5" input: "report_uninitialized_variables_1/VarIsInitializedOp_6" input: "report_uninitialized_variables_1/VarIsInitializedOp_7" input: "report_uninitialized_variables_1/VarIsInitializedOp_8" input: "report_uninitialized_variables_1/VarIsInitializedOp_9" input: "report_uninitialized_variables_1/VarIsInitializedOp_10" input: "report_uninitialized_variables_1/VarIsInitializedOp_11" input: "report_uninitialized_variables_1/VarIsInitializedOp_12" input: "report_uninitialized_variables_1/VarIsInitializedOp_13" input: "report_uninitialized_variables_1/VarIsInitializedOp_14" input: "report_uninitialized_variables_1/VarIsInitializedOp_15" input: "report_uninitialized_variables_1/VarIsInitializedOp_16" input: "report_uninitialized_variables_1/VarIsInitializedOp_17" input: "report_uninitialized_variables_1/VarIsInitializedOp_18" input: "report_uninitialized_variables_1/VarIsInitializedOp_19" input: "report_uninitialized_variables_1/VarIsInitializedOp_20" input: "report_uninitialized_variables_1/VarIsInitializedOp_21" input: "report_uninitialized_variables_1/VarIsInitializedOp_22" input: "report_uninitialized_variables_1/VarIsInitializedOp_23" input: "report_uninitialized_variables_1/VarIsInitializedOp_24" input: "report_uninitialized_variables_1/VarIsInitializedOp_25" input: "report_uninitialized_variables_1/VarIsInitializedOp_26" input: "report_uninitialized_variables_1/VarIsInitializedOp_27" input: "report_uninitialized_variables_1/VarIsInitializedOp_28" input: "report_uninitialized_variables_1/VarIsInitializedOp_29" input: "report_uninitialized_variables_1/VarIsInitializedOp_30" input: "report_uninitialized_variables_1/VarIsInitializedOp_31" input: "report_uninitialized_variables_1/VarIsInitializedOp_32" input: "report_uninitialized_variables_1/VarIsInitializedOp_33" input: "report_uninitialized_variables_1/VarIsInitializedOp_34" input: "report_uninitialized_variables_1/VarIsInitializedOp_35" input: "report_uninitialized_variables_1/VarIsInitializedOp_36" input: "report_uninitialized_variables_1/VarIsInitializedOp_37" input: "report_uninitialized_variables_1/VarIsInitializedOp_38" input: "report_uninitialized_variables_1/VarIsInitializedOp_39" input: "report_uninitialized_variables_1/VarIsInitializedOp_40" input: "report_uninitialized_variables_1/VarIsInitializedOp_41" input: "report_uninitialized_variables_1/VarIsInitializedOp_42" input: "report_uninitialized_variables_1/VarIsInitializedOp_43" input: "report_uninitialized_variables_1/VarIsInitializedOp_44" input: "report_uninitialized_variables_1/VarIsInitializedOp_45" input: "report_uninitialized_variables_1/VarIsInitializedOp_46" input: "report_uninitialized_variables_1/VarIsInitializedOp_47" input: "report_uninitialized_variables_1/VarIsInitializedOp_48" input: "report_uninitialized_variables_1/VarIsInitializedOp_49" input: "report_uninitialized_variables_1/VarIsInitializedOp_50" input: "report_uninitialized_variables_1/VarIsInitializedOp_51" input: "report_uninitialized_variables_1/VarIsInitializedOp_52" input: "report_uninitialized_variables_1/VarIsInitializedOp_53" input: "report_uninitialized_variables_1/VarIsInitializedOp_54" input: "report_uninitialized_variables_1/VarIsInitializedOp_55" input: "report_uninitialized_variables_1/VarIsInitializedOp_56" input: "report_uninitialized_variables_1/VarIsInitializedOp_57" input: "report_uninitialized_variables_1/VarIsInitializedOp_58" input: "report_uninitialized_variables_1/VarIsInitializedOp_59" input: "report_uninitialized_variables_1/VarIsInitializedOp_60" input: "report_uninitialized_variables_1/VarIsInitializedOp_61" input: "report_uninitialized_variables_1/VarIsInitializedOp_62" input: "report_uninitialized_variables_1/VarIsInitializedOp_63" input: "report_uninitialized_variables_1/VarIsInitializedOp_64" input: "report_uninitialized_variables_1/VarIsInitializedOp_65" input: "report_uninitialized_variables_1/VarIsInitializedOp_66" input: "report_uninitialized_variables_1/VarIsInitializedOp_67" input: "report_uninitialized_variables_1/VarIsInitializedOp_68" input: "report_uninitialized_variables_1/VarIsInitializedOp_69" input: "report_uninitialized_variables_1/VarIsInitializedOp_70" input: "report_uninitialized_variables_1/VarIsInitializedOp_71" input: "report_uninitialized_variables_1/VarIsInitializedOp_72" input: "report_uninitialized_variables_1/VarIsInitializedOp_73" input: "report_uninitialized_variables_1/VarIsInitializedOp_74" input: "report_uninitialized_variables_1/VarIsInitializedOp_75" input: "report_uninitialized_variables_1/VarIsInitializedOp_76" input: "report_uninitialized_variables_1/VarIsInitializedOp_77" input: "report_uninitialized_variables_1/VarIsInitializedOp_78" input: "report_uninitialized_variables_1/VarIsInitializedOp_79" input: "report_uninitialized_variables_1/VarIsInitializedOp_80" input: "report_uninitialized_variables_1/VarIsInitializedOp_81" input: "report_uninitialized_variables_1/VarIsInitializedOp_82" input: "report_uninitialized_variables_1/VarIsInitializedOp_83" input: "report_uninitialized_variables_1/VarIsInitializedOp_84" input: "report_uninitialized_variables_1/VarIsInitializedOp_85" input: "report_uninitialized_variables_1/VarIsInitializedOp_86" input: "report_uninitialized_variables_1/VarIsInitializedOp_87" input: "report_uninitialized_variables_1/VarIsInitializedOp_88" input: "report_uninitialized_variables_1/VarIsInitializedOp_89" input: "report_uninitialized_variables_1/VarIsInitializedOp_90" input: "report_uninitialized_variables_1/VarIsInitializedOp_91" input: "report_uninitialized_variables_1/VarIsInitializedOp_92" input: "report_uninitialized_variables_1/VarIsInitializedOp_93" input: "report_uninitialized_variables_1/VarIsInitializedOp_94" input: "report_uninitialized_variables_1/VarIsInitializedOp_95" input: "report_uninitialized_variables_1/VarIsInitializedOp_96" input: "report_uninitialized_variables_1/VarIsInitializedOp_97" input: "report_uninitialized_variables_1/VarIsInitializedOp_98" input: "report_uninitialized_variables_1/VarIsInitializedOp_99" input: "report_uninitialized_variables_1/VarIsInitializedOp_100" input: "report_uninitialized_variables_1/VarIsInitializedOp_101" input: "report_uninitialized_variables_1/VarIsInitializedOp_102" input: "report_uninitialized_variables_1/VarIsInitializedOp_103" input: "report_uninitialized_variables_1/VarIsInitializedOp_104" input: "report_uninitialized_variables_1/VarIsInitializedOp_105" input: "report_uninitialized_variables_1/VarIsInitializedOp_106" input: "report_uninitialized_variables_1/VarIsInitializedOp_107" input: "report_uninitialized_variables_1/VarIsInitializedOp_108" input: "report_uninitialized_variables_1/VarIsInitializedOp_109" input: "report_uninitialized_variables_1/VarIsInitializedOp_110" input: "report_uninitialized_variables_1/VarIsInitializedOp_111" input: "report_uninitialized_variables_1/VarIsInitializedOp_112" input: "report_uninitialized_variables_1/VarIsInitializedOp_113" input: "report_uninitialized_variables_1/VarIsInitializedOp_114" input: "report_uninitialized_variables_1/VarIsInitializedOp_115" input: "report_uninitialized_variables_1/VarIsInitializedOp_116" input: "report_uninitialized_variables_1/VarIsInitializedOp_117" input: "report_uninitialized_variables_1/VarIsInitializedOp_118" input: "report_uninitialized_variables_1/VarIsInitializedOp_119" input: "report_uninitialized_variables_1/VarIsInitializedOp_120" input: "report_uninitialized_variables_1/VarIsInitializedOp_121" input: "report_uninitialized_variables_1/VarIsInitializedOp_122" input: "report_uninitialized_variables_1/VarIsInitializedOp_123" input: "report_uninitialized_variables_1/VarIsInitializedOp_124" input: "report_uninitialized_variables_1/VarIsInitializedOp_125" input: "report_uninitialized_variables_1/VarIsInitializedOp_126" input: "report_uninitialized_variables_1/VarIsInitializedOp_127" input: "report_uninitialized_variables_1/VarIsInitializedOp_128" input: "report_uninitialized_variables_1/VarIsInitializedOp_129" input: "report_uninitialized_variables_1/VarIsInitializedOp_130" input: "report_uninitialized_variables_1/VarIsInitializedOp_131" input: "report_uninitialized_variables_1/VarIsInitializedOp_132" input: "report_uninitialized_variables_1/VarIsInitializedOp_133" input: "report_uninitialized_variables_1/VarIsInitializedOp_134" input: "report_uninitialized_variables_1/VarIsInitializedOp_135" input: "report_uninitialized_variables_1/VarIsInitializedOp_136" input: "report_uninitialized_variables_1/VarIsInitializedOp_137" input: "report_uninitialized_variables_1/VarIsInitializedOp_138" input: "report_uninitialized_variables_1/VarIsInitializedOp_139" input: "report_uninitialized_variables_1/VarIsInitializedOp_140" input: "report_uninitialized_variables_1/VarIsInitializedOp_141" input: "report_uninitialized_variables_1/VarIsInitializedOp_142" input: "report_uninitialized_variables_1/VarIsInitializedOp_143" input: "report_uninitialized_variables_1/VarIsInitializedOp_144" input: "report_uninitialized_variables_1/VarIsInitializedOp_145" input: "report_uninitialized_variables_1/VarIsInitializedOp_146" input: "report_uninitialized_variables_1/VarIsInitializedOp_147" input: "report_uninitialized_variables_1/VarIsInitializedOp_148" input: "report_uninitialized_variables_1/VarIsInitializedOp_149" input: "report_uninitialized_variables_1/VarIsInitializedOp_150" input: "report_uninitialized_variables_1/VarIsInitializedOp_151" input: "report_uninitialized_variables_1/VarIsInitializedOp_152" input: "report_uninitialized_variables_1/VarIsInitializedOp_153" input: "report_uninitialized_variables_1/VarIsInitializedOp_154" input: "report_uninitialized_variables_1/VarIsInitializedOp_155" input: "report_uninitialized_variables_1/VarIsInitializedOp_156" input: "report_uninitialized_variables_1/VarIsInitializedOp_157" input: "report_uninitialized_variables_1/VarIsInitializedOp_158" input: "report_uninitialized_variables_1/VarIsInitializedOp_159" input: "report_uninitialized_variables_1/VarIsInitializedOp_160" input: "report_uninitialized_variables_1/VarIsInitializedOp_161" input: "report_uninitialized_variables_1/VarIsInitializedOp_162" input: "report_uninitialized_variables_1/VarIsInitializedOp_163" input: "report_uninitialized_variables_1/VarIsInitializedOp_164" input: "report_uninitialized_variables_1/VarIsInitializedOp_165" input: "report_uninitialized_variables_1/VarIsInitializedOp_166" input: "report_uninitialized_variables_1/VarIsInitializedOp_167" input: "report_uninitialized_variables_1/VarIsInitializedOp_168" input: "report_uninitialized_variables_1/VarIsInitializedOp_169" input: "report_uninitialized_variables_1/VarIsInitializedOp_170" input: "report_uninitialized_variables_1/VarIsInitializedOp_171" input: "report_uninitialized_variables_1/VarIsInitializedOp_172" input: "report_uninitialized_variables_1/VarIsInitializedOp_173" input: "report_uninitialized_variables_1/VarIsInitializedOp_174" input: "report_uninitialized_variables_1/VarIsInitializedOp_175" input: "report_uninitialized_variables_1/VarIsInitializedOp_176" input: "report_uninitialized_variables_1/VarIsInitializedOp_177" input: "report_uninitialized_variables_1/VarIsInitializedOp_178" input: "report_uninitialized_variables_1/VarIsInitializedOp_179" input: "report_uninitialized_variables_1/VarIsInitializedOp_180" input: "report_uninitialized_variables_1/VarIsInitializedOp_181" input: "report_uninitialized_variables_1/VarIsInitializedOp_182" input: "report_uninitialized_variables_1/VarIsInitializedOp_183" input: "report_uninitialized_variables_1/VarIsInitializedOp_184" input: "report_uninitialized_variables_1/VarIsInitializedOp_185" input: "report_uninitialized_variables_1/VarIsInitializedOp_186" input: "report_uninitialized_variables_1/VarIsInitializedOp_187" input: "report_uninitialized_variables_1/VarIsInitializedOp_188" input: "report_uninitialized_variables_1/VarIsInitializedOp_189" input: "report_uninitialized_variables_1/VarIsInitializedOp_190" input: "report_uninitialized_variables_1/VarIsInitializedOp_191" input: "report_uninitialized_variables_1/VarIsInitializedOp_192" input: "report_uninitialized_variables_1/VarIsInitializedOp_193" input: "report_uninitialized_variables_1/VarIsInitializedOp_194" input: "report_uninitialized_variables_1/VarIsInitializedOp_195" input: "report_uninitialized_variables_1/VarIsInitializedOp_196" input: "report_uninitialized_variables_1/VarIsInitializedOp_197" input: "report_uninitialized_variables_1/VarIsInitializedOp_198" input: "report_uninitialized_variables_1/VarIsInitializedOp_199" input: "report_uninitialized_variables_1/VarIsInitializedOp_200" input: "report_uninitialized_variables_1/VarIsInitializedOp_201" input: "report_uninitialized_variables_1/VarIsInitializedOp_202" input: "report_uninitialized_variables_1/VarIsInitializedOp_203" input: "report_uninitialized_variables_1/VarIsInitializedOp_204" input: "report_uninitialized_variables_1/VarIsInitializedOp_205" input: "report_uninitialized_variables_1/VarIsInitializedOp_206" input: "report_uninitialized_variables_1/VarIsInitializedOp_207" input: "report_uninitialized_variables_1/VarIsInitializedOp_208" input: "report_uninitialized_variables_1/VarIsInitializedOp_209" input: "report_uninitialized_variables_1/VarIsInitializedOp_210" input: "report_uninitialized_variables_1/VarIsInitializedOp_211" input: "report_uninitialized_variables_1/VarIsInitializedOp_212" input: "report_uninitialized_variables_1/VarIsInitializedOp_213" input: "report_uninitialized_variables_1/VarIsInitializedOp_214" input: "report_uninitialized_variables_1/VarIsInitializedOp_215" input: "report_uninitialized_variables_1/VarIsInitializedOp_216" input: "report_uninitialized_variables_1/VarIsInitializedOp_217" input: "report_uninitialized_variables_1/VarIsInitializedOp_218" input: "report_uninitialized_variables_1/VarIsInitializedOp_219" input: "report_uninitialized_variables_1/VarIsInitializedOp_220" input: "report_uninitialized_variables_1/VarIsInitializedOp_221" input: "report_uninitialized_variables_1/VarIsInitializedOp_222" input: "report_uninitialized_variables_1/VarIsInitializedOp_223" input: "report_uninitialized_variables_1/VarIsInitializedOp_224" input: "report_uninitialized_variables_1/VarIsInitializedOp_225" input: "report_uninitialized_variables_1/VarIsInitializedOp_226" input: "report_uninitialized_variables_1/VarIsInitializedOp_227" input: "report_uninitialized_variables_1/VarIsInitializedOp_228" input: "report_uninitialized_variables_1/VarIsInitializedOp_229" input: "report_uninitialized_variables_1/VarIsInitializedOp_230" input: "report_uninitialized_variables_1/VarIsInitializedOp_231" input: "report_uninitialized_variables_1/VarIsInitializedOp_232" input: "report_uninitialized_variables_1/VarIsInitializedOp_233" input: "report_uninitialized_variables_1/VarIsInitializedOp_234" input: "report_uninitialized_variables_1/VarIsInitializedOp_235" input: "report_uninitialized_variables_1/VarIsInitializedOp_236" input: "report_uninitialized_variables_1/VarIsInitializedOp_237" input: "report_uninitialized_variables_1/VarIsInitializedOp_238" input: "report_uninitialized_variables_1/VarIsInitializedOp_239" input: "report_uninitialized_variables_1/VarIsInitializedOp_240" input: "report_uninitialized_variables_1/VarIsInitializedOp_241" input: "report_uninitialized_variables_1/VarIsInitializedOp_242" input: "report_uninitialized_variables_1/VarIsInitializedOp_243" input: "report_uninitialized_variables_1/VarIsInitializedOp_244" input: "report_uninitialized_variables_1/VarIsInitializedOp_245" input: "report_uninitialized_variables_1/VarIsInitializedOp_246" input: "report_uninitialized_variables_1/VarIsInitializedOp_247" input: "report_uninitialized_variables_1/VarIsInitializedOp_248" input: "report_uninitialized_variables_1/VarIsInitializedOp_249" input: "report_uninitialized_variables_1/VarIsInitializedOp_250" input: "report_uninitialized_variables_1/VarIsInitializedOp_251" input: "report_uninitialized_variables_1/VarIsInitializedOp_252" input: "report_uninitialized_variables_1/VarIsInitializedOp_253" input: "report_uninitialized_variables_1/VarIsInitializedOp_254" input: "report_uninitialized_variables_1/VarIsInitializedOp_255" input: "report_uninitialized_variables_1/VarIsInitializedOp_256" input: "report_uninitialized_variables_1/VarIsInitializedOp_257" input: "report_uninitialized_variables_1/VarIsInitializedOp_258" input: "report_uninitialized_variables_1/VarIsInitializedOp_259" input: "report_uninitialized_variables_1/VarIsInitializedOp_260" input: "report_uninitialized_variables_1/VarIsInitializedOp_261" input: "report_uninitialized_variables_1/VarIsInitializedOp_262" input: "report_uninitialized_variables_1/VarIsInitializedOp_263" input: "report_uninitialized_variables_1/VarIsInitializedOp_264" input: "report_uninitialized_variables_1/VarIsInitializedOp_265" input: "report_uninitialized_variables_1/VarIsInitializedOp_266" input: "report_uninitialized_variables_1/VarIsInitializedOp_267" input: "report_uninitialized_variables_1/VarIsInitializedOp_268" input: "report_uninitialized_variables_1/VarIsInitializedOp_269" input: "report_uninitialized_variables_1/VarIsInitializedOp_270" input: "report_uninitialized_variables_1/VarIsInitializedOp_271" input: "report_uninitialized_variables_1/VarIsInitializedOp_272" input: "report_uninitialized_variables_1/VarIsInitializedOp_273" input: "report_uninitialized_variables_1/VarIsInitializedOp_274" input: "report_uninitialized_variables_1/VarIsInitializedOp_275" input: "report_uninitialized_variables_1/VarIsInitializedOp_276" input: "report_uninitialized_variables_1/VarIsInitializedOp_277" input: "report_uninitialized_variables_1/VarIsInitializedOp_278" input: "report_uninitialized_variables_1/VarIsInitializedOp_279" input: "report_uninitialized_variables_1/VarIsInitializedOp_280" input: "report_uninitialized_variables_1/VarIsInitializedOp_281" input: "report_uninitialized_variables_1/VarIsInitializedOp_282" input: "report_uninitialized_variables_1/VarIsInitializedOp_283" input: "report_uninitialized_variables_1/VarIsInitializedOp_284" input: "report_uninitialized_variables_1/VarIsInitializedOp_285" input: "report_uninitialized_variables_1/VarIsInitializedOp_286" input: "report_uninitialized_variables_1/VarIsInitializedOp_287" input: "report_uninitialized_variables_1/VarIsInitializedOp_288" input: "report_uninitialized_variables_1/VarIsInitializedOp_289" input: "report_uninitialized_variables_1/VarIsInitializedOp_290" input: "report_uninitialized_variables_1/VarIsInitializedOp_291" input: "report_uninitialized_variables_1/VarIsInitializedOp_292" input: "report_uninitialized_variables_1/VarIsInitializedOp_293" input: "report_uninitialized_variables_1/VarIsInitializedOp_294" input: "report_uninitialized_variables_1/VarIsInitializedOp_295" input: "report_uninitialized_variables_1/VarIsInitializedOp_296" input: "report_uninitialized_variables_1/VarIsInitializedOp_297" input: "report_uninitialized_variables_1/VarIsInitializedOp_298" input: "report_uninitialized_variables_1/VarIsInitializedOp_299" input: "report_uninitialized_variables_1/VarIsInitializedOp_300" input: "report_uninitialized_variables_1/VarIsInitializedOp_301" input: "report_uninitialized_variables_1/VarIsInitializedOp_302" input: "report_uninitialized_variables_1/VarIsInitializedOp_303" input: "report_uninitialized_variables_1/VarIsInitializedOp_304" input: "report_uninitialized_variables_1/VarIsInitializedOp_305" input: "report_uninitialized_variables_1/VarIsInitializedOp_306" input: "report_uninitialized_variables_1/VarIsInitializedOp_307" input: "report_uninitialized_variables_1/VarIsInitializedOp_308" input: "report_uninitialized_variables_1/VarIsInitializedOp_309" input: "report_uninitialized_variables_1/VarIsInitializedOp_310" input: "report_uninitialized_variables_1/VarIsInitializedOp_311" input: "report_uninitialized_variables_1/VarIsInitializedOp_312" input: "report_uninitialized_variables_1/VarIsInitializedOp_313" input: "report_uninitialized_variables_1/VarIsInitializedOp_314" input: "report_uninitialized_variables_1/VarIsInitializedOp_315" input: "report_uninitialized_variables_1/VarIsInitializedOp_316" input: "report_uninitialized_variables_1/VarIsInitializedOp_317" input: "report_uninitialized_variables_1/VarIsInitializedOp_318" input: "report_uninitialized_variables_1/VarIsInitializedOp_319" input: "report_uninitialized_variables_1/VarIsInitializedOp_320" input: "report_uninitialized_variables_1/VarIsInitializedOp_321" input: "report_uninitialized_variables_1/VarIsInitializedOp_322" input: "report_uninitialized_variables_1/VarIsInitializedOp_323" input: "report_uninitialized_variables_1/VarIsInitializedOp_324" input: "report_uninitialized_variables_1/VarIsInitializedOp_325" input: "report_uninitialized_variables_1/VarIsInitializedOp_326" input: "report_uninitialized_variables_1/VarIsInitializedOp_327" input: "report_uninitialized_variables_1/VarIsInitializedOp_328" input: "report_uninitialized_variables_1/VarIsInitializedOp_329" input: "report_uninitialized_variables_1/VarIsInitializedOp_330" input: "report_uninitialized_variables_1/VarIsInitializedOp_331" input: "report_uninitialized_variables_1/VarIsInitializedOp_332" input: "report_uninitialized_variables_1/VarIsInitializedOp_333" input: "report_uninitialized_variables_1/VarIsInitializedOp_334" input: "report_uninitialized_variables_1/VarIsInitializedOp_335" input: "report_uninitialized_variables_1/VarIsInitializedOp_336" input: "report_uninitialized_variables_1/VarIsInitializedOp_337" input: "report_uninitialized_variables_1/VarIsInitializedOp_338" input: "report_uninitialized_variables_1/VarIsInitializedOp_339" input: "report_uninitialized_variables_1/VarIsInitializedOp_340" input: "report_uninitialized_variables_1/VarIsInitializedOp_341" input: "report_uninitialized_variables_1/VarIsInitializedOp_342" input: "report_uninitialized_variables_1/VarIsInitializedOp_343" input: "report_uninitialized_variables_1/VarIsInitializedOp_344" input: "report_uninitialized_variables_1/VarIsInitializedOp_345" input: "report_uninitialized_variables_1/VarIsInitializedOp_346" input: "report_uninitialized_variables_1/VarIsInitializedOp_347" input: "report_uninitialized_variables_1/VarIsInitializedOp_348" input: "report_uninitialized_variables_1/VarIsInitializedOp_349" input: "report_uninitialized_variables_1/VarIsInitializedOp_350" input: "report_uninitialized_variables_1/VarIsInitializedOp_351" input: "report_uninitialized_variables_1/VarIsInitializedOp_352" input: "report_uninitialized_variables_1/VarIsInitializedOp_353" input: "report_uninitialized_variables_1/VarIsInitializedOp_354" input: "report_uninitialized_variables_1/VarIsInitializedOp_355" input: "report_uninitialized_variables_1/VarIsInitializedOp_356" input: "report_uninitialized_variables_1/VarIsInitializedOp_357" input: "report_uninitialized_variables_1/VarIsInitializedOp_358" input: "report_uninitialized_variables_1/VarIsInitializedOp_359" input: "report_uninitialized_variables_1/VarIsInitializedOp_360" input: "report_uninitialized_variables_1/VarIsInitializedOp_361" input: "report_uninitialized_variables_1/VarIsInitializedOp_362" input: "report_uninitialized_variables_1/VarIsInitializedOp_363" input: "report_uninitialized_variables_1/VarIsInitializedOp_364" input: "report_uninitialized_variables_1/VarIsInitializedOp_365" input: "report_uninitialized_variables_1/VarIsInitializedOp_366" input: "report_uninitialized_variables_1/VarIsInitializedOp_367" input: "report_uninitialized_variables_1/VarIsInitializedOp_368" input: "report_uninitialized_variables_1/VarIsInitializedOp_369" input: "report_uninitialized_variables_1/VarIsInitializedOp_370" input: "report_uninitialized_variables_1/VarIsInitializedOp_371" input: "report_uninitialized_variables_1/VarIsInitializedOp_372" input: "report_uninitialized_variables_1/VarIsInitializedOp_373" input: "report_uninitialized_variables_1/VarIsInitializedOp_374" input: "report_uninitialized_variables_1/VarIsInitializedOp_375" input: "report_uninitialized_variables_1/VarIsInitializedOp_376" input: "report_uninitialized_variables_1/VarIsInitializedOp_377" input: "report_uninitialized_variables_1/VarIsInitializedOp_378" input: "report_uninitialized_variables_1/VarIsInitializedOp_379" input: "report_uninitialized_variables_1/VarIsInitializedOp_380" input: "report_uninitialized_variables_1/VarIsInitializedOp_381" input: "report_uninitialized_variables_1/VarIsInitializedOp_382" input: "report_uninitialized_variables_1/VarIsInitializedOp_383" input: "report_uninitialized_variables_1/VarIsInitializedOp_384" input: "report_uninitialized_variables_1/VarIsInitializedOp_385" input: "report_uninitialized_variables_1/VarIsInitializedOp_386" input: "report_uninitialized_variables_1/VarIsInitializedOp_387" input: "report_uninitialized_variables_1/VarIsInitializedOp_388" input: "report_uninitialized_variables_1/VarIsInitializedOp_389" input: "report_uninitialized_variables_1/VarIsInitializedOp_390" input: "report_uninitialized_variables_1/VarIsInitializedOp_391" input: "report_uninitialized_variables_1/VarIsInitializedOp_392" input: "report_uninitialized_variables_1/VarIsInitializedOp_393" input: "report_uninitialized_variables_1/VarIsInitializedOp_394" input: "report_uninitialized_variables_1/VarIsInitializedOp_395" input: "report_uninitialized_variables_1/VarIsInitializedOp_396" input: "report_uninitialized_variables_1/VarIsInitializedOp_397" input: "report_uninitialized_variables_1/VarIsInitializedOp_398" input: "report_uninitialized_variables_1/VarIsInitializedOp_399" input: "report_uninitialized_variables_1/VarIsInitializedOp_400" input: "report_uninitialized_variables_1/VarIsInitializedOp_401" input: "report_uninitialized_variables_1/VarIsInitializedOp_402" input: "report_uninitialized_variables_1/VarIsInitializedOp_403" input: "report_uninitialized_variables_1/VarIsInitializedOp_404" input: "report_uninitialized_variables_1/VarIsInitializedOp_405" input: "report_uninitialized_variables_1/VarIsInitializedOp_406" input: "report_uninitialized_variables_1/VarIsInitializedOp_407" input: "report_uninitialized_variables_1/VarIsInitializedOp_408" input: "report_uninitialized_variables_1/VarIsInitializedOp_409" input: "report_uninitialized_variables_1/VarIsInitializedOp_410" input: "report_uninitialized_variables_1/VarIsInitializedOp_411" input: "report_uninitialized_variables_1/VarIsInitializedOp_412" input: "report_uninitialized_variables_1/VarIsInitializedOp_413" input: "report_uninitialized_variables_1/VarIsInitializedOp_414" input: "report_uninitialized_variables_1/VarIsInitializedOp_415" input: "report_uninitialized_variables_1/VarIsInitializedOp_416" input: "report_uninitialized_variables_1/VarIsInitializedOp_417" input: "report_uninitialized_variables_1/VarIsInitializedOp_418" input: "report_uninitialized_variables_1/VarIsInitializedOp_419" input: "report_uninitialized_variables_1/VarIsInitializedOp_420" input: "report_uninitialized_variables_1/VarIsInitializedOp_421" input: "report_uninitialized_variables_1/VarIsInitializedOp_422" input: "report_uninitialized_variables_1/VarIsInitializedOp_423" input: "report_uninitialized_variables_1/VarIsInitializedOp_424" input: "report_uninitialized_variables_1/VarIsInitializedOp_425" input: "report_uninitialized_variables_1/VarIsInitializedOp_426" input: "report_uninitialized_variables_1/VarIsInitializedOp_427" input: "report_uninitialized_variables_1/VarIsInitializedOp_428" input: "report_uninitialized_variables_1/VarIsInitializedOp_429" input: "report_uninitialized_variables_1/VarIsInitializedOp_430" input: "report_uninitialized_variables_1/VarIsInitializedOp_431" input: "report_uninitialized_variables_1/VarIsInitializedOp_432" input: "report_uninitialized_variables_1/VarIsInitializedOp_433" input: "report_uninitialized_variables_1/VarIsInitializedOp_434" input: "report_uninitialized_variables_1/VarIsInitializedOp_435" input: "report_uninitialized_variables_1/VarIsInitializedOp_436" input: "report_uninitialized_variables_1/VarIsInitializedOp_437" input: "report_uninitialized_variables_1/VarIsInitializedOp_438" input: "report_uninitialized_variables_1/VarIsInitializedOp_439" input: "report_uninitialized_variables_1/VarIsInitializedOp_440" input: "report_uninitialized_variables_1/VarIsInitializedOp_441" input: "report_uninitialized_variables_1/VarIsInitializedOp_442" input: "report_uninitialized_variables_1/VarIsInitializedOp_443" input: "report_uninitialized_variables_1/VarIsInitializedOp_444" input: "report_uninitialized_variables_1/VarIsInitializedOp_445" input: "report_uninitialized_variables_1/VarIsInitializedOp_446" input: "report_uninitialized_variables_1/VarIsInitializedOp_447" input: "report_uninitialized_variables_1/VarIsInitializedOp_448" input: "report_uninitialized_variables_1/VarIsInitializedOp_449" input: "report_uninitialized_variables_1/VarIsInitializedOp_450" input: "report_uninitialized_variables_1/VarIsInitializedOp_451" input: "report_uninitialized_variables_1/VarIsInitializedOp_452" input: "report_uninitialized_variables_1/VarIsInitializedOp_453" input: "report_uninitialized_variables_1/VarIsInitializedOp_454" input: "report_uninitialized_variables_1/VarIsInitializedOp_455" input: "report_uninitialized_variables_1/VarIsInitializedOp_456" input: "report_uninitialized_variables_1/VarIsInitializedOp_457" input: "report_uninitialized_variables_1/VarIsInitializedOp_458" input: "report_uninitialized_variables_1/VarIsInitializedOp_459" input: "report_uninitialized_variables_1/VarIsInitializedOp_460" input: "report_uninitialized_variables_1/VarIsInitializedOp_461" input: "report_uninitialized_variables_1/VarIsInitializedOp_462" input: "report_uninitialized_variables_1/VarIsInitializedOp_463" input: "report_uninitialized_variables_1/VarIsInitializedOp_464" input: "report_uninitialized_variables_1/VarIsInitializedOp_465" input: "report_uninitialized_variables_1/VarIsInitializedOp_466" input: "report_uninitialized_variables_1/VarIsInitializedOp_467" input: "report_uninitialized_variables_1/VarIsInitializedOp_468" input: "report_uninitialized_variables_1/VarIsInitializedOp_469" input: "report_uninitialized_variables_1/VarIsInitializedOp_470" input: "report_uninitialized_variables_1/VarIsInitializedOp_471" input: "report_uninitialized_variables_1/VarIsInitializedOp_472" input: "report_uninitialized_variables_1/VarIsInitializedOp_473" input: "report_uninitialized_variables_1/VarIsInitializedOp_474" input: "report_uninitialized_variables_1/VarIsInitializedOp_475" input: "report_uninitialized_variables_1/VarIsInitializedOp_476" input: "report_uninitialized_variables_1/VarIsInitializedOp_477" input: "report_uninitialized_variables_1/VarIsInitializedOp_478" input: "report_uninitialized_variables_1/VarIsInitializedOp_479" input: "report_uninitialized_variables_1/VarIsInitializedOp_480" input: "report_uninitialized_variables_1/VarIsInitializedOp_481" input: "report_uninitialized_variables_1/VarIsInitializedOp_482" input: "report_uninitialized_variables_1/VarIsInitializedOp_483" input: "report_uninitialized_variables_1/VarIsInitializedOp_484" input: "report_uninitialized_variables_1/VarIsInitializedOp_485" input: "report_uninitialized_variables_1/VarIsInitializedOp_486" input: "report_uninitialized_variables_1/VarIsInitializedOp_487" input: "report_uninitialized_variables_1/VarIsInitializedOp_488" input: "report_uninitialized_variables_1/VarIsInitializedOp_489" input: "report_uninitialized_variables_1/VarIsInitializedOp_490" input: "report_uninitialized_variables_1/VarIsInitializedOp_491" input: "report_uninitialized_variables_1/VarIsInitializedOp_492" input: "report_uninitialized_variables_1/VarIsInitializedOp_493" input: "report_uninitialized_variables_1/VarIsInitializedOp_494" input: "report_uninitialized_variables_1/VarIsInitializedOp_495" input: "report_uninitialized_variables_1/VarIsInitializedOp_496" input: "report_uninitialized_variables_1/VarIsInitializedOp_497" input: "report_uninitialized_variables_1/VarIsInitializedOp_498" input: "report_uninitialized_variables_1/VarIsInitializedOp_499" input: "report_uninitialized_variables_1/VarIsInitializedOp_500" input: "report_uninitialized_variables_1/VarIsInitializedOp_501" input: "report_uninitialized_variables_1/VarIsInitializedOp_502" input: "report_uninitialized_variables_1/VarIsInitializedOp_503" input: "report_uninitialized_variables_1/VarIsInitializedOp_504" input: "report_uninitialized_variables_1/VarIsInitializedOp_505" input: "report_uninitialized_variables_1/VarIsInitializedOp_506" input: "report_uninitialized_variables_1/VarIsInitializedOp_507" input: "report_uninitialized_variables_1/VarIsInitializedOp_508" input: "report_uninitialized_variables_1/VarIsInitializedOp_509" input: "report_uninitialized_variables_1/VarIsInitializedOp_510" input: "report_uninitialized_variables_1/VarIsInitializedOp_511" input: "report_uninitialized_variables_1/VarIsInitializedOp_512" input: "report_uninitialized_variables_1/VarIsInitializedOp_513" input: "report_uninitialized_variables_1/VarIsInitializedOp_514" input: "report_uninitialized_variables_1/VarIsInitializedOp_515" input: "report_uninitialized_variables_1/VarIsInitializedOp_516" input: "report_uninitialized_variables_1/VarIsInitializedOp_517" input: "report_uninitialized_variables_1/VarIsInitializedOp_518" input: "report_uninitialized_variables_1/VarIsInitializedOp_519" input: "report_uninitialized_variables_1/VarIsInitializedOp_520" input: "report_uninitialized_variables_1/VarIsInitializedOp_521" input: "report_uninitialized_variables_1/VarIsInitializedOp_522" input: "report_uninitialized_variables_1/VarIsInitializedOp_523" input: "report_uninitialized_variables_1/VarIsInitializedOp_524" input: "report_uninitialized_variables_1/VarIsInitializedOp_525" input: "report_uninitialized_variables_1/VarIsInitializedOp_526" input: "report_uninitialized_variables_1/VarIsInitializedOp_527" input: "report_uninitialized_variables_1/VarIsInitializedOp_528" input: "report_uninitialized_variables_1/VarIsInitializedOp_529" input: "report_uninitialized_variables_1/VarIsInitializedOp_530" input: "report_uninitialized_variables_1/VarIsInitializedOp_531" input: "report_uninitialized_variables_1/VarIsInitializedOp_532" input: "report_uninitialized_variables_1/VarIsInitializedOp_533" input: "report_uninitialized_variables_1/VarIsInitializedOp_534" input: "report_uninitialized_variables_1/VarIsInitializedOp_535" input: "report_uninitialized_variables_1/VarIsInitializedOp_536" input: "report_uninitialized_variables_1/VarIsInitializedOp_537" input: "report_uninitialized_variables_1/VarIsInitializedOp_538" input: "report_uninitialized_variables_1/VarIsInitializedOp_539" input: "report_uninitialized_variables_1/VarIsInitializedOp_540" input: "report_uninitialized_variables_1/VarIsInitializedOp_541" input: "report_uninitialized_variables_1/VarIsInitializedOp_542" input: "report_uninitialized_variables_1/VarIsInitializedOp_543" input: "report_uninitialized_variables_1/VarIsInitializedOp_544" input: "report_uninitialized_variables_1/VarIsInitializedOp_545" input: "report_uninitialized_variables_1/VarIsInitializedOp_546" input: "report_uninitialized_variables_1/VarIsInitializedOp_547" input: "report_uninitialized_variables_1/VarIsInitializedOp_548" input: "report_uninitialized_variables_1/VarIsInitializedOp_549" input: "report_uninitialized_variables_1/VarIsInitializedOp_550" input: "report_uninitialized_variables_1/VarIsInitializedOp_551" input: "report_uninitialized_variables_1/VarIsInitializedOp_552" input: "report_uninitialized_variables_1/VarIsInitializedOp_553" input: "report_uninitialized_variables_1/VarIsInitializedOp_554" input: "report_uninitialized_variables_1/VarIsInitializedOp_555" input: "report_uninitialized_variables_1/VarIsInitializedOp_556" input: "report_uninitialized_variables_1/VarIsInitializedOp_557" input: "report_uninitialized_variables_1/VarIsInitializedOp_558" input: "report_uninitialized_variables_1/VarIsInitializedOp_559" input: "report_uninitialized_variables_1/VarIsInitializedOp_560" input: "report_uninitialized_variables_1/VarIsInitializedOp_561" input: "report_uninitialized_variables_1/VarIsInitializedOp_562" input: "report_uninitialized_variables_1/VarIsInitializedOp_563" input: "report_uninitialized_variables_1/VarIsInitializedOp_564" input: "report_uninitialized_variables_1/VarIsInitializedOp_565" input: "report_uninitialized_variables_1/VarIsInitializedOp_566" input: "report_uninitialized_variables_1/VarIsInitializedOp_567" input: "report_uninitialized_variables_1/VarIsInitializedOp_568" input: "report_uninitialized_variables_1/VarIsInitializedOp_569" input: "report_uninitialized_variables_1/VarIsInitializedOp_570" input: "report_uninitialized_variables_1/VarIsInitializedOp_571" input: "report_uninitialized_variables_1/VarIsInitializedOp_572" input: "report_uninitialized_variables_1/VarIsInitializedOp_573" input: "report_uninitialized_variables_1/VarIsInitializedOp_574" input: "report_uninitialized_variables_1/VarIsInitializedOp_575" input: "report_uninitialized_variables_1/VarIsInitializedOp_576" input: "report_uninitialized_variables_1/VarIsInitializedOp_577" input: "report_uninitialized_variables_1/VarIsInitializedOp_578" input: "report_uninitialized_variables_1/VarIsInitializedOp_579" input: "report_uninitialized_variables_1/VarIsInitializedOp_580" input: "report_uninitialized_variables_1/VarIsInitializedOp_581" input: "report_uninitialized_variables_1/VarIsInitializedOp_582" input: "report_uninitialized_variables_1/VarIsInitializedOp_583" input: "report_uninitialized_variables_1/VarIsInitializedOp_584" input: "report_uninitialized_variables_1/VarIsInitializedOp_585" input: "report_uninitialized_variables_1/VarIsInitializedOp_586" input: "report_uninitialized_variables_1/VarIsInitializedOp_587" input: "report_uninitialized_variables_1/VarIsInitializedOp_588" input: "report_uninitialized_variables_1/VarIsInitializedOp_589" input: "report_uninitialized_variables_1/VarIsInitializedOp_590" input: "report_uninitialized_variables_1/VarIsInitializedOp_591" input: "report_uninitialized_variables_1/VarIsInitializedOp_592" input: "report_uninitialized_variables_1/VarIsInitializedOp_593" input: "report_uninitialized_variables_1/VarIsInitializedOp_594" input: "report_uninitialized_variables_1/VarIsInitializedOp_595" input: "report_uninitialized_variables_1/VarIsInitializedOp_596" input: "report_uninitialized_variables_1/VarIsInitializedOp_597" input: "report_uninitialized_variables_1/VarIsInitializedOp_598" input: "report_uninitialized_variables_1/VarIsInitializedOp_599" input: "report_uninitialized_variables_1/VarIsInitializedOp_600" input: "report_uninitialized_variables_1/VarIsInitializedOp_601" input: "report_uninitialized_variables_1/VarIsInitializedOp_602" input: "report_uninitialized_variables_1/VarIsInitializedOp_603" device: "/device:CPU:0" attr { key: "N" value { i: 604 } } attr { key: "T" value { type: DT_BOOL } } attr { key: "_output_shapes" value { list { shape { dim { size: 604 } } } } } attr { key: "axis" value { i: 0 } } } node { name: "report_uninitialized_variables_1/LogicalNot" op: "LogicalNot" input: "report_uninitialized_variables_1/stack" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 604 } } } } } } node { name: "report_uninitialized_variables_1/Const" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 604 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 604 } } string_val: "global_step" string_val: "bert/embeddings/word_embeddings" string_val: "bert/embeddings/token_type_embeddings" string_val: "bert/embeddings/position_embeddings" string_val: "bert/embeddings/layer_normalization/gamma" string_val: "bert/embeddings/layer_normalization/beta" string_val: "bert/encoder/layer_0/attention/self/query/kernel" string_val: "bert/encoder/layer_0/attention/self/query/bias" string_val: "bert/encoder/layer_0/attention/self/key/kernel" string_val: "bert/encoder/layer_0/attention/self/key/bias" string_val: "bert/encoder/layer_0/attention/self/value/kernel" string_val: "bert/encoder/layer_0/attention/self/value/bias" string_val: "bert/encoder/layer_0/attention/output/dense/kernel" string_val: "bert/encoder/layer_0/attention/output/dense/bias" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta" string_val: "bert/encoder/layer_0/intermediate/dense/kernel" string_val: "bert/encoder/layer_0/intermediate/dense/bias" string_val: "bert/encoder/layer_0/output/dense/kernel" string_val: "bert/encoder/layer_0/output/dense/bias" string_val: "bert/encoder/layer_0/output/layer_normalization_2/gamma" string_val: "bert/encoder/layer_0/output/layer_normalization_2/beta" string_val: "bert/encoder/layer_1/attention/self/query/kernel" string_val: "bert/encoder/layer_1/attention/self/query/bias" string_val: "bert/encoder/layer_1/attention/self/key/kernel" string_val: "bert/encoder/layer_1/attention/self/key/bias" string_val: "bert/encoder/layer_1/attention/self/value/kernel" string_val: "bert/encoder/layer_1/attention/self/value/bias" string_val: "bert/encoder/layer_1/attention/output/dense/kernel" string_val: "bert/encoder/layer_1/attention/output/dense/bias" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta" string_val: "bert/encoder/layer_1/intermediate/dense/kernel" string_val: "bert/encoder/layer_1/intermediate/dense/bias" string_val: "bert/encoder/layer_1/output/dense/kernel" string_val: "bert/encoder/layer_1/output/dense/bias" string_val: "bert/encoder/layer_1/output/layer_normalization_4/gamma" string_val: "bert/encoder/layer_1/output/layer_normalization_4/beta" string_val: "bert/encoder/layer_2/attention/self/query/kernel" string_val: "bert/encoder/layer_2/attention/self/query/bias" string_val: "bert/encoder/layer_2/attention/self/key/kernel" string_val: "bert/encoder/layer_2/attention/self/key/bias" string_val: "bert/encoder/layer_2/attention/self/value/kernel" string_val: "bert/encoder/layer_2/attention/self/value/bias" string_val: "bert/encoder/layer_2/attention/output/dense/kernel" string_val: "bert/encoder/layer_2/attention/output/dense/bias" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta" string_val: "bert/encoder/layer_2/intermediate/dense/kernel" string_val: "bert/encoder/layer_2/intermediate/dense/bias" string_val: "bert/encoder/layer_2/output/dense/kernel" string_val: "bert/encoder/layer_2/output/dense/bias" string_val: "bert/encoder/layer_2/output/layer_normalization_6/gamma" string_val: "bert/encoder/layer_2/output/layer_normalization_6/beta" string_val: "bert/encoder/layer_3/attention/self/query/kernel" string_val: "bert/encoder/layer_3/attention/self/query/bias" string_val: "bert/encoder/layer_3/attention/self/key/kernel" string_val: "bert/encoder/layer_3/attention/self/key/bias" string_val: "bert/encoder/layer_3/attention/self/value/kernel" string_val: "bert/encoder/layer_3/attention/self/value/bias" string_val: "bert/encoder/layer_3/attention/output/dense/kernel" string_val: "bert/encoder/layer_3/attention/output/dense/bias" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta" string_val: "bert/encoder/layer_3/intermediate/dense/kernel" string_val: "bert/encoder/layer_3/intermediate/dense/bias" string_val: "bert/encoder/layer_3/output/dense/kernel" string_val: "bert/encoder/layer_3/output/dense/bias" string_val: "bert/encoder/layer_3/output/layer_normalization_8/gamma" string_val: "bert/encoder/layer_3/output/layer_normalization_8/beta" string_val: "bert/encoder/layer_4/attention/self/query/kernel" string_val: "bert/encoder/layer_4/attention/self/query/bias" string_val: "bert/encoder/layer_4/attention/self/key/kernel" string_val: "bert/encoder/layer_4/attention/self/key/bias" string_val: "bert/encoder/layer_4/attention/self/value/kernel" string_val: "bert/encoder/layer_4/attention/self/value/bias" string_val: "bert/encoder/layer_4/attention/output/dense/kernel" string_val: "bert/encoder/layer_4/attention/output/dense/bias" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta" string_val: "bert/encoder/layer_4/intermediate/dense/kernel" string_val: "bert/encoder/layer_4/intermediate/dense/bias" string_val: "bert/encoder/layer_4/output/dense/kernel" string_val: "bert/encoder/layer_4/output/dense/bias" string_val: "bert/encoder/layer_4/output/layer_normalization_10/gamma" string_val: "bert/encoder/layer_4/output/layer_normalization_10/beta" string_val: "bert/encoder/layer_5/attention/self/query/kernel" string_val: "bert/encoder/layer_5/attention/self/query/bias" string_val: "bert/encoder/layer_5/attention/self/key/kernel" string_val: "bert/encoder/layer_5/attention/self/key/bias" string_val: "bert/encoder/layer_5/attention/self/value/kernel" string_val: "bert/encoder/layer_5/attention/self/value/bias" string_val: "bert/encoder/layer_5/attention/output/dense/kernel" string_val: "bert/encoder/layer_5/attention/output/dense/bias" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta" string_val: "bert/encoder/layer_5/intermediate/dense/kernel" string_val: "bert/encoder/layer_5/intermediate/dense/bias" string_val: "bert/encoder/layer_5/output/dense/kernel" string_val: "bert/encoder/layer_5/output/dense/bias" string_val: "bert/encoder/layer_5/output/layer_normalization_12/gamma" string_val: "bert/encoder/layer_5/output/layer_normalization_12/beta" string_val: "bert/encoder/layer_6/attention/self/query/kernel" string_val: "bert/encoder/layer_6/attention/self/query/bias" string_val: "bert/encoder/layer_6/attention/self/key/kernel" string_val: "bert/encoder/layer_6/attention/self/key/bias" string_val: "bert/encoder/layer_6/attention/self/value/kernel" string_val: "bert/encoder/layer_6/attention/self/value/bias" string_val: "bert/encoder/layer_6/attention/output/dense/kernel" string_val: "bert/encoder/layer_6/attention/output/dense/bias" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta" string_val: "bert/encoder/layer_6/intermediate/dense/kernel" string_val: "bert/encoder/layer_6/intermediate/dense/bias" string_val: "bert/encoder/layer_6/output/dense/kernel" string_val: "bert/encoder/layer_6/output/dense/bias" string_val: "bert/encoder/layer_6/output/layer_normalization_14/gamma" string_val: "bert/encoder/layer_6/output/layer_normalization_14/beta" string_val: "bert/encoder/layer_7/attention/self/query/kernel" string_val: "bert/encoder/layer_7/attention/self/query/bias" string_val: "bert/encoder/layer_7/attention/self/key/kernel" string_val: "bert/encoder/layer_7/attention/self/key/bias" string_val: "bert/encoder/layer_7/attention/self/value/kernel" string_val: "bert/encoder/layer_7/attention/self/value/bias" string_val: "bert/encoder/layer_7/attention/output/dense/kernel" string_val: "bert/encoder/layer_7/attention/output/dense/bias" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta" string_val: "bert/encoder/layer_7/intermediate/dense/kernel" string_val: "bert/encoder/layer_7/intermediate/dense/bias" string_val: "bert/encoder/layer_7/output/dense/kernel" string_val: "bert/encoder/layer_7/output/dense/bias" string_val: "bert/encoder/layer_7/output/layer_normalization_16/gamma" string_val: "bert/encoder/layer_7/output/layer_normalization_16/beta" string_val: "bert/encoder/layer_8/attention/self/query/kernel" string_val: "bert/encoder/layer_8/attention/self/query/bias" string_val: "bert/encoder/layer_8/attention/self/key/kernel" string_val: "bert/encoder/layer_8/attention/self/key/bias" string_val: "bert/encoder/layer_8/attention/self/value/kernel" string_val: "bert/encoder/layer_8/attention/self/value/bias" string_val: "bert/encoder/layer_8/attention/output/dense/kernel" string_val: "bert/encoder/layer_8/attention/output/dense/bias" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta" string_val: "bert/encoder/layer_8/intermediate/dense/kernel" string_val: "bert/encoder/layer_8/intermediate/dense/bias" string_val: "bert/encoder/layer_8/output/dense/kernel" string_val: "bert/encoder/layer_8/output/dense/bias" string_val: "bert/encoder/layer_8/output/layer_normalization_18/gamma" string_val: "bert/encoder/layer_8/output/layer_normalization_18/beta" string_val: "bert/encoder/layer_9/attention/self/query/kernel" string_val: "bert/encoder/layer_9/attention/self/query/bias" string_val: "bert/encoder/layer_9/attention/self/key/kernel" string_val: "bert/encoder/layer_9/attention/self/key/bias" string_val: "bert/encoder/layer_9/attention/self/value/kernel" string_val: "bert/encoder/layer_9/attention/self/value/bias" string_val: "bert/encoder/layer_9/attention/output/dense/kernel" string_val: "bert/encoder/layer_9/attention/output/dense/bias" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta" string_val: "bert/encoder/layer_9/intermediate/dense/kernel" string_val: "bert/encoder/layer_9/intermediate/dense/bias" string_val: "bert/encoder/layer_9/output/dense/kernel" string_val: "bert/encoder/layer_9/output/dense/bias" string_val: "bert/encoder/layer_9/output/layer_normalization_20/gamma" string_val: "bert/encoder/layer_9/output/layer_normalization_20/beta" string_val: "bert/encoder/layer_10/attention/self/query/kernel" string_val: "bert/encoder/layer_10/attention/self/query/bias" string_val: "bert/encoder/layer_10/attention/self/key/kernel" string_val: "bert/encoder/layer_10/attention/self/key/bias" string_val: "bert/encoder/layer_10/attention/self/value/kernel" string_val: "bert/encoder/layer_10/attention/self/value/bias" string_val: "bert/encoder/layer_10/attention/output/dense/kernel" string_val: "bert/encoder/layer_10/attention/output/dense/bias" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta" string_val: "bert/encoder/layer_10/intermediate/dense/kernel" string_val: "bert/encoder/layer_10/intermediate/dense/bias" string_val: "bert/encoder/layer_10/output/dense/kernel" string_val: "bert/encoder/layer_10/output/dense/bias" string_val: "bert/encoder/layer_10/output/layer_normalization_22/gamma" string_val: "bert/encoder/layer_10/output/layer_normalization_22/beta" string_val: "bert/encoder/layer_11/attention/self/query/kernel" string_val: "bert/encoder/layer_11/attention/self/query/bias" string_val: "bert/encoder/layer_11/attention/self/key/kernel" string_val: "bert/encoder/layer_11/attention/self/key/bias" string_val: "bert/encoder/layer_11/attention/self/value/kernel" string_val: "bert/encoder/layer_11/attention/self/value/bias" string_val: "bert/encoder/layer_11/attention/output/dense/kernel" string_val: "bert/encoder/layer_11/attention/output/dense/bias" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta" string_val: "bert/encoder/layer_11/intermediate/dense/kernel" string_val: "bert/encoder/layer_11/intermediate/dense/bias" string_val: "bert/encoder/layer_11/output/dense/kernel" string_val: "bert/encoder/layer_11/output/dense/bias" string_val: "bert/encoder/layer_11/output/layer_normalization_24/gamma" string_val: "bert/encoder/layer_11/output/layer_normalization_24/beta" string_val: "bert/pooler/dense/kernel" string_val: "bert/pooler/dense/bias" string_val: "output_weights" string_val: "output_bias" string_val: "bert/embeddings/word_embeddings/adam_m" string_val: "bert/embeddings/word_embeddings/adam_v" string_val: "bert/embeddings/token_type_embeddings/adam_m" string_val: "bert/embeddings/token_type_embeddings/adam_v" string_val: "bert/embeddings/position_embeddings/adam_m" string_val: "bert/embeddings/position_embeddings/adam_v" string_val: "bert/embeddings/layer_normalization/gamma/adam_m" string_val: "bert/embeddings/layer_normalization/gamma/adam_v" string_val: "bert/embeddings/layer_normalization/beta/adam_m" string_val: "bert/embeddings/layer_normalization/beta/adam_v" string_val: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" string_val: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_0/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/output/dense/bias/adam_m" string_val: "bert/encoder/layer_0/output/dense/bias/adam_v" string_val: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" string_val: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" string_val: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" string_val: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" string_val: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" string_val: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_1/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/output/dense/bias/adam_m" string_val: "bert/encoder/layer_1/output/dense/bias/adam_v" string_val: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" string_val: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" string_val: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" string_val: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" string_val: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" string_val: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_2/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/output/dense/bias/adam_m" string_val: "bert/encoder/layer_2/output/dense/bias/adam_v" string_val: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" string_val: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" string_val: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" string_val: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" string_val: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" string_val: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_3/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/output/dense/bias/adam_m" string_val: "bert/encoder/layer_3/output/dense/bias/adam_v" string_val: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" string_val: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" string_val: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" string_val: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" string_val: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" string_val: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_4/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/output/dense/bias/adam_m" string_val: "bert/encoder/layer_4/output/dense/bias/adam_v" string_val: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" string_val: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" string_val: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" string_val: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" string_val: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" string_val: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_5/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/output/dense/bias/adam_m" string_val: "bert/encoder/layer_5/output/dense/bias/adam_v" string_val: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" string_val: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" string_val: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" string_val: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" string_val: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" string_val: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_6/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/output/dense/bias/adam_m" string_val: "bert/encoder/layer_6/output/dense/bias/adam_v" string_val: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" string_val: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" string_val: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" string_val: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" string_val: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" string_val: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_7/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/output/dense/bias/adam_m" string_val: "bert/encoder/layer_7/output/dense/bias/adam_v" string_val: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" string_val: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" string_val: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" string_val: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" string_val: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" string_val: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_8/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/output/dense/bias/adam_m" string_val: "bert/encoder/layer_8/output/dense/bias/adam_v" string_val: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" string_val: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" string_val: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" string_val: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" string_val: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" string_val: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_9/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/output/dense/bias/adam_m" string_val: "bert/encoder/layer_9/output/dense/bias/adam_v" string_val: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" string_val: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" string_val: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" string_val: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" string_val: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" string_val: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_10/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/output/dense/bias/adam_m" string_val: "bert/encoder/layer_10/output/dense/bias/adam_v" string_val: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" string_val: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" string_val: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" string_val: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" string_val: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" string_val: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_11/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/output/dense/bias/adam_m" string_val: "bert/encoder/layer_11/output/dense/bias/adam_v" string_val: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" string_val: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" string_val: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" string_val: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" string_val: "bert/pooler/dense/kernel/adam_m" string_val: "bert/pooler/dense/kernel/adam_v" string_val: "bert/pooler/dense/bias/adam_m" string_val: "bert/pooler/dense/bias/adam_v" string_val: "output_weights/adam_m" string_val: "output_weights/adam_v" string_val: "output_bias/adam_m" string_val: "output_bias/adam_v" } } } } node { name: "report_uninitialized_variables_1/boolean_mask/Shape" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 604 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack_2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice" op: "StridedSlice" input: "report_uninitialized_variables_1/boolean_mask/Shape" input: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack" input: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack_1" input: "report_uninitialized_variables_1/boolean_mask/strided_slice/stack_2" device: "/device:CPU:0" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 0 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "report_uninitialized_variables_1/boolean_mask/Prod/reduction_indices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/Prod" op: "Prod" input: "report_uninitialized_variables_1/boolean_mask/strided_slice" input: "report_uninitialized_variables_1/boolean_mask/Prod/reduction_indices" device: "/device:CPU:0" attr { key: "T" value { type: DT_INT32 } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "keep_dims" value { b: false } } } node { name: "report_uninitialized_variables_1/boolean_mask/Shape_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 604 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack_2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice_1" op: "StridedSlice" input: "report_uninitialized_variables_1/boolean_mask/Shape_1" input: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack" input: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack_1" input: "report_uninitialized_variables_1/boolean_mask/strided_slice_1/stack_2" device: "/device:CPU:0" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "begin_mask" value { i: 1 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 0 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "report_uninitialized_variables_1/boolean_mask/Shape_2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 604 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack_1" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 0 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack_2" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: 1 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/strided_slice_2" op: "StridedSlice" input: "report_uninitialized_variables_1/boolean_mask/Shape_2" input: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack" input: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack_1" input: "report_uninitialized_variables_1/boolean_mask/strided_slice_2/stack_2" device: "/device:CPU:0" attr { key: "Index" value { type: DT_INT32 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "begin_mask" value { i: 0 } } attr { key: "ellipsis_mask" value { i: 0 } } attr { key: "end_mask" value { i: 1 } } attr { key: "new_axis_mask" value { i: 0 } } attr { key: "shrink_axis_mask" value { i: 0 } } } node { name: "report_uninitialized_variables_1/boolean_mask/concat/values_1" op: "Pack" input: "report_uninitialized_variables_1/boolean_mask/Prod" device: "/device:CPU:0" attr { key: "N" value { i: 1 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "axis" value { i: 0 } } } node { name: "report_uninitialized_variables_1/boolean_mask/concat/axis" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/concat" op: "ConcatV2" input: "report_uninitialized_variables_1/boolean_mask/strided_slice_1" input: "report_uninitialized_variables_1/boolean_mask/concat/values_1" input: "report_uninitialized_variables_1/boolean_mask/strided_slice_2" input: "report_uninitialized_variables_1/boolean_mask/concat/axis" device: "/device:CPU:0" attr { key: "N" value { i: 3 } } attr { key: "T" value { type: DT_INT32 } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } } node { name: "report_uninitialized_variables_1/boolean_mask/Reshape" op: "Reshape" input: "report_uninitialized_variables_1/Const" input: "report_uninitialized_variables_1/boolean_mask/concat" device: "/device:CPU:0" attr { key: "T" value { type: DT_STRING } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 604 } } } } } } node { name: "report_uninitialized_variables_1/boolean_mask/Reshape_1/shape" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { dim { size: 1 } } int_val: -1 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/Reshape_1" op: "Reshape" input: "report_uninitialized_variables_1/LogicalNot" input: "report_uninitialized_variables_1/boolean_mask/Reshape_1/shape" device: "/device:CPU:0" attr { key: "T" value { type: DT_BOOL } } attr { key: "Tshape" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 604 } } } } } } node { name: "report_uninitialized_variables_1/boolean_mask/Where" op: "Where" input: "report_uninitialized_variables_1/boolean_mask/Reshape_1" device: "/device:CPU:0" attr { key: "T" value { type: DT_BOOL } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } dim { size: 1 } } } } } } node { name: "report_uninitialized_variables_1/boolean_mask/Squeeze" op: "Squeeze" input: "report_uninitialized_variables_1/boolean_mask/Where" device: "/device:CPU:0" attr { key: "T" value { type: DT_INT64 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } } } } attr { key: "squeeze_dims" value { list { i: 1 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/GatherV2/axis" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "report_uninitialized_variables_1/boolean_mask/GatherV2" op: "GatherV2" input: "report_uninitialized_variables_1/boolean_mask/Reshape" input: "report_uninitialized_variables_1/boolean_mask/Squeeze" input: "report_uninitialized_variables_1/boolean_mask/GatherV2/axis" device: "/device:CPU:0" attr { key: "Taxis" value { type: DT_INT32 } } attr { key: "Tindices" value { type: DT_INT64 } } attr { key: "Tparams" value { type: DT_STRING } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } } } } attr { key: "batch_dims" value { i: 0 } } } node { name: "report_uninitialized_resources_1/Const" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { } } } } } } node { name: "concat_1/axis" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "concat_1" op: "ConcatV2" input: "report_uninitialized_variables_1/boolean_mask/GatherV2" input: "report_uninitialized_resources_1/Const" input: "concat_1/axis" attr { key: "N" value { i: 2 } } attr { key: "T" value { type: DT_STRING } } attr { key: "Tidx" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: -1 } } } } } } node { name: "init_2" op: "NoOp" attr { key: "_has_manual_control_dependencies" value { b: true } } } node { name: "init_all_tables" op: "NoOp" attr { key: "_has_manual_control_dependencies" value { b: true } } } node { name: "init_3" op: "NoOp" attr { key: "_has_manual_control_dependencies" value { b: true } } } node { name: "group_deps_3" op: "NoOp" input: "^init_2" input: "^init_3" input: "^init_all_tables" } node { name: "Merge/MergeSummary" op: "MergeSummary" input: "loss_1" attr { key: "N" value { i: 1 } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "save/filename/input" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "model" } } } } node { name: "save/filename" op: "PlaceholderWithDefault" input: "save/filename/input" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "shape" value { shape { } } } } node { name: "save/Const" op: "PlaceholderWithDefault" input: "save/filename" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "shape" value { shape { } } } } node { name: "save/StaticRegexFullMatch" op: "StaticRegexFullMatch" input: "save/Const" device: "/device:CPU:*" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "pattern" value { s: "^s3://.*" } } } node { name: "save/Const_1" op: "Const" device: "/device:CPU:*" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: ".part" } } } } node { name: "save/Const_2" op: "Const" device: "/device:CPU:*" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "_temp/part" } } } } node { name: "save/Select" op: "Select" input: "save/StaticRegexFullMatch" input: "save/Const_1" input: "save/Const_2" device: "/device:CPU:*" attr { key: "T" value { type: DT_STRING } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "save/StringJoin" op: "StringJoin" input: "save/Const" input: "save/Select" device: "/device:CPU:*" attr { key: "N" value { i: 2 } } attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "separator" value { s: "" } } } node { name: "save/num_shards" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 1 } } } } node { name: "save/ShardedFilename/shard" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT32 } } attr { key: "value" value { tensor { dtype: DT_INT32 tensor_shape { } int_val: 0 } } } } node { name: "save/ShardedFilename" op: "ShardedFilename" input: "save/StringJoin" input: "save/ShardedFilename/shard" input: "save/num_shards" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "save/SaveV2/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 604 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 604 } } string_val: "bert/embeddings/layer_normalization/beta" string_val: "bert/embeddings/layer_normalization/beta/adam_m" string_val: "bert/embeddings/layer_normalization/beta/adam_v" string_val: "bert/embeddings/layer_normalization/gamma" string_val: "bert/embeddings/layer_normalization/gamma/adam_m" string_val: "bert/embeddings/layer_normalization/gamma/adam_v" string_val: "bert/embeddings/position_embeddings" string_val: "bert/embeddings/position_embeddings/adam_m" string_val: "bert/embeddings/position_embeddings/adam_v" string_val: "bert/embeddings/token_type_embeddings" string_val: "bert/embeddings/token_type_embeddings/adam_m" string_val: "bert/embeddings/token_type_embeddings/adam_v" string_val: "bert/embeddings/word_embeddings" string_val: "bert/embeddings/word_embeddings/adam_m" string_val: "bert/embeddings/word_embeddings/adam_v" string_val: "bert/encoder/layer_0/attention/output/dense/bias" string_val: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_0/attention/output/dense/kernel" string_val: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" string_val: "bert/encoder/layer_0/attention/self/key/bias" string_val: "bert/encoder/layer_0/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/key/kernel" string_val: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/query/bias" string_val: "bert/encoder/layer_0/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/query/kernel" string_val: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/value/bias" string_val: "bert/encoder/layer_0/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/value/kernel" string_val: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_0/intermediate/dense/bias" string_val: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_0/intermediate/dense/kernel" string_val: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/output/dense/bias" string_val: "bert/encoder/layer_0/output/dense/bias/adam_m" string_val: "bert/encoder/layer_0/output/dense/bias/adam_v" string_val: "bert/encoder/layer_0/output/dense/kernel" string_val: "bert/encoder/layer_0/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/output/layer_normalization_2/beta" string_val: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" string_val: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" string_val: "bert/encoder/layer_0/output/layer_normalization_2/gamma" string_val: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" string_val: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" string_val: "bert/encoder/layer_1/attention/output/dense/bias" string_val: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_1/attention/output/dense/kernel" string_val: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" string_val: "bert/encoder/layer_1/attention/self/key/bias" string_val: "bert/encoder/layer_1/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/key/kernel" string_val: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/query/bias" string_val: "bert/encoder/layer_1/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/query/kernel" string_val: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/value/bias" string_val: "bert/encoder/layer_1/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/value/kernel" string_val: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_1/intermediate/dense/bias" string_val: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_1/intermediate/dense/kernel" string_val: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/output/dense/bias" string_val: "bert/encoder/layer_1/output/dense/bias/adam_m" string_val: "bert/encoder/layer_1/output/dense/bias/adam_v" string_val: "bert/encoder/layer_1/output/dense/kernel" string_val: "bert/encoder/layer_1/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/output/layer_normalization_4/beta" string_val: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" string_val: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" string_val: "bert/encoder/layer_1/output/layer_normalization_4/gamma" string_val: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" string_val: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" string_val: "bert/encoder/layer_10/attention/output/dense/bias" string_val: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_10/attention/output/dense/kernel" string_val: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" string_val: "bert/encoder/layer_10/attention/self/key/bias" string_val: "bert/encoder/layer_10/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/key/kernel" string_val: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/query/bias" string_val: "bert/encoder/layer_10/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/query/kernel" string_val: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/value/bias" string_val: "bert/encoder/layer_10/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/value/kernel" string_val: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_10/intermediate/dense/bias" string_val: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_10/intermediate/dense/kernel" string_val: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/output/dense/bias" string_val: "bert/encoder/layer_10/output/dense/bias/adam_m" string_val: "bert/encoder/layer_10/output/dense/bias/adam_v" string_val: "bert/encoder/layer_10/output/dense/kernel" string_val: "bert/encoder/layer_10/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/output/layer_normalization_22/beta" string_val: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" string_val: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" string_val: "bert/encoder/layer_10/output/layer_normalization_22/gamma" string_val: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" string_val: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" string_val: "bert/encoder/layer_11/attention/output/dense/bias" string_val: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_11/attention/output/dense/kernel" string_val: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" string_val: "bert/encoder/layer_11/attention/self/key/bias" string_val: "bert/encoder/layer_11/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/key/kernel" string_val: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/query/bias" string_val: "bert/encoder/layer_11/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/query/kernel" string_val: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/value/bias" string_val: "bert/encoder/layer_11/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/value/kernel" string_val: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_11/intermediate/dense/bias" string_val: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_11/intermediate/dense/kernel" string_val: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/output/dense/bias" string_val: "bert/encoder/layer_11/output/dense/bias/adam_m" string_val: "bert/encoder/layer_11/output/dense/bias/adam_v" string_val: "bert/encoder/layer_11/output/dense/kernel" string_val: "bert/encoder/layer_11/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/output/layer_normalization_24/beta" string_val: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" string_val: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" string_val: "bert/encoder/layer_11/output/layer_normalization_24/gamma" string_val: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" string_val: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" string_val: "bert/encoder/layer_2/attention/output/dense/bias" string_val: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_2/attention/output/dense/kernel" string_val: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" string_val: "bert/encoder/layer_2/attention/self/key/bias" string_val: "bert/encoder/layer_2/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/key/kernel" string_val: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/query/bias" string_val: "bert/encoder/layer_2/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/query/kernel" string_val: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/value/bias" string_val: "bert/encoder/layer_2/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/value/kernel" string_val: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_2/intermediate/dense/bias" string_val: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_2/intermediate/dense/kernel" string_val: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/output/dense/bias" string_val: "bert/encoder/layer_2/output/dense/bias/adam_m" string_val: "bert/encoder/layer_2/output/dense/bias/adam_v" string_val: "bert/encoder/layer_2/output/dense/kernel" string_val: "bert/encoder/layer_2/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/output/layer_normalization_6/beta" string_val: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" string_val: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" string_val: "bert/encoder/layer_2/output/layer_normalization_6/gamma" string_val: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" string_val: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" string_val: "bert/encoder/layer_3/attention/output/dense/bias" string_val: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_3/attention/output/dense/kernel" string_val: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" string_val: "bert/encoder/layer_3/attention/self/key/bias" string_val: "bert/encoder/layer_3/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/key/kernel" string_val: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/query/bias" string_val: "bert/encoder/layer_3/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/query/kernel" string_val: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/value/bias" string_val: "bert/encoder/layer_3/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/value/kernel" string_val: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_3/intermediate/dense/bias" string_val: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_3/intermediate/dense/kernel" string_val: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/output/dense/bias" string_val: "bert/encoder/layer_3/output/dense/bias/adam_m" string_val: "bert/encoder/layer_3/output/dense/bias/adam_v" string_val: "bert/encoder/layer_3/output/dense/kernel" string_val: "bert/encoder/layer_3/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/output/layer_normalization_8/beta" string_val: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" string_val: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" string_val: "bert/encoder/layer_3/output/layer_normalization_8/gamma" string_val: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" string_val: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" string_val: "bert/encoder/layer_4/attention/output/dense/bias" string_val: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_4/attention/output/dense/kernel" string_val: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" string_val: "bert/encoder/layer_4/attention/self/key/bias" string_val: "bert/encoder/layer_4/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/key/kernel" string_val: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/query/bias" string_val: "bert/encoder/layer_4/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/query/kernel" string_val: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/value/bias" string_val: "bert/encoder/layer_4/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/value/kernel" string_val: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_4/intermediate/dense/bias" string_val: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_4/intermediate/dense/kernel" string_val: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/output/dense/bias" string_val: "bert/encoder/layer_4/output/dense/bias/adam_m" string_val: "bert/encoder/layer_4/output/dense/bias/adam_v" string_val: "bert/encoder/layer_4/output/dense/kernel" string_val: "bert/encoder/layer_4/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/output/layer_normalization_10/beta" string_val: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" string_val: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" string_val: "bert/encoder/layer_4/output/layer_normalization_10/gamma" string_val: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" string_val: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" string_val: "bert/encoder/layer_5/attention/output/dense/bias" string_val: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_5/attention/output/dense/kernel" string_val: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" string_val: "bert/encoder/layer_5/attention/self/key/bias" string_val: "bert/encoder/layer_5/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/key/kernel" string_val: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/query/bias" string_val: "bert/encoder/layer_5/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/query/kernel" string_val: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/value/bias" string_val: "bert/encoder/layer_5/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/value/kernel" string_val: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_5/intermediate/dense/bias" string_val: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_5/intermediate/dense/kernel" string_val: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/output/dense/bias" string_val: "bert/encoder/layer_5/output/dense/bias/adam_m" string_val: "bert/encoder/layer_5/output/dense/bias/adam_v" string_val: "bert/encoder/layer_5/output/dense/kernel" string_val: "bert/encoder/layer_5/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/output/layer_normalization_12/beta" string_val: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" string_val: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" string_val: "bert/encoder/layer_5/output/layer_normalization_12/gamma" string_val: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" string_val: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" string_val: "bert/encoder/layer_6/attention/output/dense/bias" string_val: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_6/attention/output/dense/kernel" string_val: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" string_val: "bert/encoder/layer_6/attention/self/key/bias" string_val: "bert/encoder/layer_6/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/key/kernel" string_val: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/query/bias" string_val: "bert/encoder/layer_6/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/query/kernel" string_val: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/value/bias" string_val: "bert/encoder/layer_6/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/value/kernel" string_val: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_6/intermediate/dense/bias" string_val: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_6/intermediate/dense/kernel" string_val: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/output/dense/bias" string_val: "bert/encoder/layer_6/output/dense/bias/adam_m" string_val: "bert/encoder/layer_6/output/dense/bias/adam_v" string_val: "bert/encoder/layer_6/output/dense/kernel" string_val: "bert/encoder/layer_6/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/output/layer_normalization_14/beta" string_val: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" string_val: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" string_val: "bert/encoder/layer_6/output/layer_normalization_14/gamma" string_val: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" string_val: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" string_val: "bert/encoder/layer_7/attention/output/dense/bias" string_val: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_7/attention/output/dense/kernel" string_val: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" string_val: "bert/encoder/layer_7/attention/self/key/bias" string_val: "bert/encoder/layer_7/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/key/kernel" string_val: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/query/bias" string_val: "bert/encoder/layer_7/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/query/kernel" string_val: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/value/bias" string_val: "bert/encoder/layer_7/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/value/kernel" string_val: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_7/intermediate/dense/bias" string_val: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_7/intermediate/dense/kernel" string_val: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/output/dense/bias" string_val: "bert/encoder/layer_7/output/dense/bias/adam_m" string_val: "bert/encoder/layer_7/output/dense/bias/adam_v" string_val: "bert/encoder/layer_7/output/dense/kernel" string_val: "bert/encoder/layer_7/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/output/layer_normalization_16/beta" string_val: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" string_val: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" string_val: "bert/encoder/layer_7/output/layer_normalization_16/gamma" string_val: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" string_val: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" string_val: "bert/encoder/layer_8/attention/output/dense/bias" string_val: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_8/attention/output/dense/kernel" string_val: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" string_val: "bert/encoder/layer_8/attention/self/key/bias" string_val: "bert/encoder/layer_8/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/key/kernel" string_val: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/query/bias" string_val: "bert/encoder/layer_8/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/query/kernel" string_val: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/value/bias" string_val: "bert/encoder/layer_8/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/value/kernel" string_val: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_8/intermediate/dense/bias" string_val: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_8/intermediate/dense/kernel" string_val: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/output/dense/bias" string_val: "bert/encoder/layer_8/output/dense/bias/adam_m" string_val: "bert/encoder/layer_8/output/dense/bias/adam_v" string_val: "bert/encoder/layer_8/output/dense/kernel" string_val: "bert/encoder/layer_8/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/output/layer_normalization_18/beta" string_val: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" string_val: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" string_val: "bert/encoder/layer_8/output/layer_normalization_18/gamma" string_val: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" string_val: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" string_val: "bert/encoder/layer_9/attention/output/dense/bias" string_val: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_9/attention/output/dense/kernel" string_val: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" string_val: "bert/encoder/layer_9/attention/self/key/bias" string_val: "bert/encoder/layer_9/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/key/kernel" string_val: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/query/bias" string_val: "bert/encoder/layer_9/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/query/kernel" string_val: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/value/bias" string_val: "bert/encoder/layer_9/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/value/kernel" string_val: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_9/intermediate/dense/bias" string_val: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_9/intermediate/dense/kernel" string_val: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/output/dense/bias" string_val: "bert/encoder/layer_9/output/dense/bias/adam_m" string_val: "bert/encoder/layer_9/output/dense/bias/adam_v" string_val: "bert/encoder/layer_9/output/dense/kernel" string_val: "bert/encoder/layer_9/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/output/layer_normalization_20/beta" string_val: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" string_val: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" string_val: "bert/encoder/layer_9/output/layer_normalization_20/gamma" string_val: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" string_val: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" string_val: "bert/pooler/dense/bias" string_val: "bert/pooler/dense/bias/adam_m" string_val: "bert/pooler/dense/bias/adam_v" string_val: "bert/pooler/dense/kernel" string_val: "bert/pooler/dense/kernel/adam_m" string_val: "bert/pooler/dense/kernel/adam_v" string_val: "global_step" string_val: "output_bias" string_val: "output_bias/adam_m" string_val: "output_bias/adam_v" string_val: "output_weights" string_val: "output_weights/adam_m" string_val: "output_weights/adam_v" } } } } node { name: "save/SaveV2/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 604 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 604 } } string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" } } } } node { name: "save/SaveV2" op: "SaveV2" input: "save/ShardedFilename" input: "save/SaveV2/tensor_names" input: "save/SaveV2/shape_and_slices" input: "bert/embeddings/layer_normalization/beta/Read/ReadVariableOp" input: "bert/embeddings/layer_normalization/beta/adam_m/Read/ReadVariableOp" input: "bert/embeddings/layer_normalization/beta/adam_v/Read/ReadVariableOp" input: "bert/embeddings/layer_normalization/gamma/Read/ReadVariableOp" input: "bert/embeddings/layer_normalization/gamma/adam_m/Read/ReadVariableOp" input: "bert/embeddings/layer_normalization/gamma/adam_v/Read/ReadVariableOp" input: "bert/embeddings/position_embeddings/Read/ReadVariableOp" input: "bert/embeddings/position_embeddings/adam_m/Read/ReadVariableOp" input: "bert/embeddings/position_embeddings/adam_v/Read/ReadVariableOp" input: "bert/embeddings/token_type_embeddings/Read/ReadVariableOp" input: "bert/embeddings/token_type_embeddings/adam_m/Read/ReadVariableOp" input: "bert/embeddings/token_type_embeddings/adam_v/Read/ReadVariableOp" input: "bert/embeddings/word_embeddings/Read/ReadVariableOp" input: "bert/embeddings/word_embeddings/adam_m/Read/ReadVariableOp" input: "bert/embeddings/word_embeddings/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_0/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_0/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_0/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_0/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/Read/ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_1/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_1/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_1/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_1/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/Read/ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_10/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_10/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_10/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_10/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/Read/ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_11/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_11/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_11/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_11/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/Read/ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_2/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_2/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_2/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_2/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/Read/ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_3/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_3/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_3/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_3/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/Read/ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_4/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_4/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_4/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_4/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/Read/ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_5/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_5/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_5/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_5/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/Read/ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_6/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_6/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_6/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_6/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/Read/ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_7/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_7/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_7/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_7/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/Read/ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_8/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_8/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_8/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_8/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/Read/ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_9/output/dense/bias/Read/ReadVariableOp" input: "bert/encoder/layer_9/output/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_9/output/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_9/output/dense/kernel/Read/ReadVariableOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/Read/ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v/Read/ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/Read/ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m/Read/ReadVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v/Read/ReadVariableOp" input: "bert/pooler/dense/bias/Read/ReadVariableOp" input: "bert/pooler/dense/bias/adam_m/Read/ReadVariableOp" input: "bert/pooler/dense/bias/adam_v/Read/ReadVariableOp" input: "bert/pooler/dense/kernel/Read/ReadVariableOp" input: "bert/pooler/dense/kernel/adam_m/Read/ReadVariableOp" input: "bert/pooler/dense/kernel/adam_v/Read/ReadVariableOp" input: "global_step/Read/ReadVariableOp" input: "output_bias/Read/ReadVariableOp" input: "output_bias/adam_m/Read/ReadVariableOp" input: "output_bias/adam_v/Read/ReadVariableOp" input: "output_weights/Read/ReadVariableOp" input: "output_weights/adam_m/Read/ReadVariableOp" input: "output_weights/adam_v/Read/ReadVariableOp" device: "/device:CPU:0" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtypes" value { list { type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_INT64 type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT } } } } node { name: "save/control_dependency" op: "Identity" input: "save/ShardedFilename" input: "^save/SaveV2" device: "/device:CPU:0" attr { key: "T" value { type: DT_STRING } } attr { key: "_class" value { list { s: "loc:@save/ShardedFilename" } } } attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "save/MergeV2Checkpoints/checkpoint_prefixes" op: "Pack" input: "save/ShardedFilename" input: "^save/control_dependency" device: "/device:CPU:0" attr { key: "N" value { i: 1 } } attr { key: "T" value { type: DT_STRING } } attr { key: "_output_shapes" value { list { shape { dim { size: 1 } } } } } attr { key: "axis" value { i: 0 } } } node { name: "save/MergeV2Checkpoints" op: "MergeV2Checkpoints" input: "save/MergeV2Checkpoints/checkpoint_prefixes" input: "save/Const" device: "/device:CPU:0" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "allow_missing_files" value { b: false } } attr { key: "delete_old_dirs" value { b: true } } } node { name: "save/Identity" op: "Identity" input: "save/Const" input: "^save/MergeV2Checkpoints" input: "^save/control_dependency" device: "/device:CPU:0" attr { key: "T" value { type: DT_STRING } } attr { key: "_output_shapes" value { list { shape { } } } } } node { name: "save/RestoreV2/tensor_names" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 604 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 604 } } string_val: "bert/embeddings/layer_normalization/beta" string_val: "bert/embeddings/layer_normalization/beta/adam_m" string_val: "bert/embeddings/layer_normalization/beta/adam_v" string_val: "bert/embeddings/layer_normalization/gamma" string_val: "bert/embeddings/layer_normalization/gamma/adam_m" string_val: "bert/embeddings/layer_normalization/gamma/adam_v" string_val: "bert/embeddings/position_embeddings" string_val: "bert/embeddings/position_embeddings/adam_m" string_val: "bert/embeddings/position_embeddings/adam_v" string_val: "bert/embeddings/token_type_embeddings" string_val: "bert/embeddings/token_type_embeddings/adam_m" string_val: "bert/embeddings/token_type_embeddings/adam_v" string_val: "bert/embeddings/word_embeddings" string_val: "bert/embeddings/word_embeddings/adam_m" string_val: "bert/embeddings/word_embeddings/adam_v" string_val: "bert/encoder/layer_0/attention/output/dense/bias" string_val: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_0/attention/output/dense/kernel" string_val: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" string_val: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" string_val: "bert/encoder/layer_0/attention/self/key/bias" string_val: "bert/encoder/layer_0/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/key/kernel" string_val: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/query/bias" string_val: "bert/encoder/layer_0/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/query/kernel" string_val: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_0/attention/self/value/bias" string_val: "bert/encoder/layer_0/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_0/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_0/attention/self/value/kernel" string_val: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_0/intermediate/dense/bias" string_val: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_0/intermediate/dense/kernel" string_val: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/output/dense/bias" string_val: "bert/encoder/layer_0/output/dense/bias/adam_m" string_val: "bert/encoder/layer_0/output/dense/bias/adam_v" string_val: "bert/encoder/layer_0/output/dense/kernel" string_val: "bert/encoder/layer_0/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_0/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_0/output/layer_normalization_2/beta" string_val: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" string_val: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" string_val: "bert/encoder/layer_0/output/layer_normalization_2/gamma" string_val: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" string_val: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" string_val: "bert/encoder/layer_1/attention/output/dense/bias" string_val: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_1/attention/output/dense/kernel" string_val: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" string_val: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" string_val: "bert/encoder/layer_1/attention/self/key/bias" string_val: "bert/encoder/layer_1/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/key/kernel" string_val: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/query/bias" string_val: "bert/encoder/layer_1/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/query/kernel" string_val: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_1/attention/self/value/bias" string_val: "bert/encoder/layer_1/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_1/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_1/attention/self/value/kernel" string_val: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_1/intermediate/dense/bias" string_val: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_1/intermediate/dense/kernel" string_val: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/output/dense/bias" string_val: "bert/encoder/layer_1/output/dense/bias/adam_m" string_val: "bert/encoder/layer_1/output/dense/bias/adam_v" string_val: "bert/encoder/layer_1/output/dense/kernel" string_val: "bert/encoder/layer_1/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_1/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_1/output/layer_normalization_4/beta" string_val: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" string_val: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" string_val: "bert/encoder/layer_1/output/layer_normalization_4/gamma" string_val: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" string_val: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" string_val: "bert/encoder/layer_10/attention/output/dense/bias" string_val: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_10/attention/output/dense/kernel" string_val: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" string_val: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" string_val: "bert/encoder/layer_10/attention/self/key/bias" string_val: "bert/encoder/layer_10/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/key/kernel" string_val: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/query/bias" string_val: "bert/encoder/layer_10/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/query/kernel" string_val: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_10/attention/self/value/bias" string_val: "bert/encoder/layer_10/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_10/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_10/attention/self/value/kernel" string_val: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_10/intermediate/dense/bias" string_val: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_10/intermediate/dense/kernel" string_val: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/output/dense/bias" string_val: "bert/encoder/layer_10/output/dense/bias/adam_m" string_val: "bert/encoder/layer_10/output/dense/bias/adam_v" string_val: "bert/encoder/layer_10/output/dense/kernel" string_val: "bert/encoder/layer_10/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_10/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_10/output/layer_normalization_22/beta" string_val: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" string_val: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" string_val: "bert/encoder/layer_10/output/layer_normalization_22/gamma" string_val: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" string_val: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" string_val: "bert/encoder/layer_11/attention/output/dense/bias" string_val: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_11/attention/output/dense/kernel" string_val: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" string_val: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" string_val: "bert/encoder/layer_11/attention/self/key/bias" string_val: "bert/encoder/layer_11/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/key/kernel" string_val: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/query/bias" string_val: "bert/encoder/layer_11/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/query/kernel" string_val: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_11/attention/self/value/bias" string_val: "bert/encoder/layer_11/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_11/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_11/attention/self/value/kernel" string_val: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_11/intermediate/dense/bias" string_val: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_11/intermediate/dense/kernel" string_val: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/output/dense/bias" string_val: "bert/encoder/layer_11/output/dense/bias/adam_m" string_val: "bert/encoder/layer_11/output/dense/bias/adam_v" string_val: "bert/encoder/layer_11/output/dense/kernel" string_val: "bert/encoder/layer_11/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_11/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_11/output/layer_normalization_24/beta" string_val: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" string_val: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" string_val: "bert/encoder/layer_11/output/layer_normalization_24/gamma" string_val: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" string_val: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" string_val: "bert/encoder/layer_2/attention/output/dense/bias" string_val: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_2/attention/output/dense/kernel" string_val: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" string_val: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" string_val: "bert/encoder/layer_2/attention/self/key/bias" string_val: "bert/encoder/layer_2/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/key/kernel" string_val: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/query/bias" string_val: "bert/encoder/layer_2/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/query/kernel" string_val: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_2/attention/self/value/bias" string_val: "bert/encoder/layer_2/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_2/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_2/attention/self/value/kernel" string_val: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_2/intermediate/dense/bias" string_val: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_2/intermediate/dense/kernel" string_val: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/output/dense/bias" string_val: "bert/encoder/layer_2/output/dense/bias/adam_m" string_val: "bert/encoder/layer_2/output/dense/bias/adam_v" string_val: "bert/encoder/layer_2/output/dense/kernel" string_val: "bert/encoder/layer_2/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_2/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_2/output/layer_normalization_6/beta" string_val: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" string_val: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" string_val: "bert/encoder/layer_2/output/layer_normalization_6/gamma" string_val: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" string_val: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" string_val: "bert/encoder/layer_3/attention/output/dense/bias" string_val: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_3/attention/output/dense/kernel" string_val: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" string_val: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" string_val: "bert/encoder/layer_3/attention/self/key/bias" string_val: "bert/encoder/layer_3/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/key/kernel" string_val: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/query/bias" string_val: "bert/encoder/layer_3/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/query/kernel" string_val: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_3/attention/self/value/bias" string_val: "bert/encoder/layer_3/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_3/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_3/attention/self/value/kernel" string_val: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_3/intermediate/dense/bias" string_val: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_3/intermediate/dense/kernel" string_val: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/output/dense/bias" string_val: "bert/encoder/layer_3/output/dense/bias/adam_m" string_val: "bert/encoder/layer_3/output/dense/bias/adam_v" string_val: "bert/encoder/layer_3/output/dense/kernel" string_val: "bert/encoder/layer_3/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_3/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_3/output/layer_normalization_8/beta" string_val: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" string_val: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" string_val: "bert/encoder/layer_3/output/layer_normalization_8/gamma" string_val: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" string_val: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" string_val: "bert/encoder/layer_4/attention/output/dense/bias" string_val: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_4/attention/output/dense/kernel" string_val: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" string_val: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" string_val: "bert/encoder/layer_4/attention/self/key/bias" string_val: "bert/encoder/layer_4/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/key/kernel" string_val: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/query/bias" string_val: "bert/encoder/layer_4/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/query/kernel" string_val: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_4/attention/self/value/bias" string_val: "bert/encoder/layer_4/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_4/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_4/attention/self/value/kernel" string_val: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_4/intermediate/dense/bias" string_val: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_4/intermediate/dense/kernel" string_val: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/output/dense/bias" string_val: "bert/encoder/layer_4/output/dense/bias/adam_m" string_val: "bert/encoder/layer_4/output/dense/bias/adam_v" string_val: "bert/encoder/layer_4/output/dense/kernel" string_val: "bert/encoder/layer_4/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_4/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_4/output/layer_normalization_10/beta" string_val: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" string_val: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" string_val: "bert/encoder/layer_4/output/layer_normalization_10/gamma" string_val: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" string_val: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" string_val: "bert/encoder/layer_5/attention/output/dense/bias" string_val: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_5/attention/output/dense/kernel" string_val: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" string_val: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" string_val: "bert/encoder/layer_5/attention/self/key/bias" string_val: "bert/encoder/layer_5/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/key/kernel" string_val: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/query/bias" string_val: "bert/encoder/layer_5/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/query/kernel" string_val: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_5/attention/self/value/bias" string_val: "bert/encoder/layer_5/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_5/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_5/attention/self/value/kernel" string_val: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_5/intermediate/dense/bias" string_val: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_5/intermediate/dense/kernel" string_val: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/output/dense/bias" string_val: "bert/encoder/layer_5/output/dense/bias/adam_m" string_val: "bert/encoder/layer_5/output/dense/bias/adam_v" string_val: "bert/encoder/layer_5/output/dense/kernel" string_val: "bert/encoder/layer_5/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_5/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_5/output/layer_normalization_12/beta" string_val: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" string_val: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" string_val: "bert/encoder/layer_5/output/layer_normalization_12/gamma" string_val: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" string_val: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" string_val: "bert/encoder/layer_6/attention/output/dense/bias" string_val: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_6/attention/output/dense/kernel" string_val: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" string_val: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" string_val: "bert/encoder/layer_6/attention/self/key/bias" string_val: "bert/encoder/layer_6/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/key/kernel" string_val: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/query/bias" string_val: "bert/encoder/layer_6/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/query/kernel" string_val: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_6/attention/self/value/bias" string_val: "bert/encoder/layer_6/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_6/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_6/attention/self/value/kernel" string_val: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_6/intermediate/dense/bias" string_val: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_6/intermediate/dense/kernel" string_val: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/output/dense/bias" string_val: "bert/encoder/layer_6/output/dense/bias/adam_m" string_val: "bert/encoder/layer_6/output/dense/bias/adam_v" string_val: "bert/encoder/layer_6/output/dense/kernel" string_val: "bert/encoder/layer_6/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_6/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_6/output/layer_normalization_14/beta" string_val: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" string_val: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" string_val: "bert/encoder/layer_6/output/layer_normalization_14/gamma" string_val: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" string_val: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" string_val: "bert/encoder/layer_7/attention/output/dense/bias" string_val: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_7/attention/output/dense/kernel" string_val: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" string_val: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" string_val: "bert/encoder/layer_7/attention/self/key/bias" string_val: "bert/encoder/layer_7/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/key/kernel" string_val: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/query/bias" string_val: "bert/encoder/layer_7/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/query/kernel" string_val: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_7/attention/self/value/bias" string_val: "bert/encoder/layer_7/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_7/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_7/attention/self/value/kernel" string_val: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_7/intermediate/dense/bias" string_val: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_7/intermediate/dense/kernel" string_val: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/output/dense/bias" string_val: "bert/encoder/layer_7/output/dense/bias/adam_m" string_val: "bert/encoder/layer_7/output/dense/bias/adam_v" string_val: "bert/encoder/layer_7/output/dense/kernel" string_val: "bert/encoder/layer_7/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_7/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_7/output/layer_normalization_16/beta" string_val: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" string_val: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" string_val: "bert/encoder/layer_7/output/layer_normalization_16/gamma" string_val: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" string_val: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" string_val: "bert/encoder/layer_8/attention/output/dense/bias" string_val: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_8/attention/output/dense/kernel" string_val: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" string_val: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" string_val: "bert/encoder/layer_8/attention/self/key/bias" string_val: "bert/encoder/layer_8/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/key/kernel" string_val: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/query/bias" string_val: "bert/encoder/layer_8/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/query/kernel" string_val: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_8/attention/self/value/bias" string_val: "bert/encoder/layer_8/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_8/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_8/attention/self/value/kernel" string_val: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_8/intermediate/dense/bias" string_val: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_8/intermediate/dense/kernel" string_val: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/output/dense/bias" string_val: "bert/encoder/layer_8/output/dense/bias/adam_m" string_val: "bert/encoder/layer_8/output/dense/bias/adam_v" string_val: "bert/encoder/layer_8/output/dense/kernel" string_val: "bert/encoder/layer_8/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_8/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_8/output/layer_normalization_18/beta" string_val: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" string_val: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" string_val: "bert/encoder/layer_8/output/layer_normalization_18/gamma" string_val: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" string_val: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" string_val: "bert/encoder/layer_9/attention/output/dense/bias" string_val: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" string_val: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" string_val: "bert/encoder/layer_9/attention/output/dense/kernel" string_val: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" string_val: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" string_val: "bert/encoder/layer_9/attention/self/key/bias" string_val: "bert/encoder/layer_9/attention/self/key/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/key/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/key/kernel" string_val: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/query/bias" string_val: "bert/encoder/layer_9/attention/self/query/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/query/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/query/kernel" string_val: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" string_val: "bert/encoder/layer_9/attention/self/value/bias" string_val: "bert/encoder/layer_9/attention/self/value/bias/adam_m" string_val: "bert/encoder/layer_9/attention/self/value/bias/adam_v" string_val: "bert/encoder/layer_9/attention/self/value/kernel" string_val: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" string_val: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" string_val: "bert/encoder/layer_9/intermediate/dense/bias" string_val: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" string_val: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" string_val: "bert/encoder/layer_9/intermediate/dense/kernel" string_val: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/output/dense/bias" string_val: "bert/encoder/layer_9/output/dense/bias/adam_m" string_val: "bert/encoder/layer_9/output/dense/bias/adam_v" string_val: "bert/encoder/layer_9/output/dense/kernel" string_val: "bert/encoder/layer_9/output/dense/kernel/adam_m" string_val: "bert/encoder/layer_9/output/dense/kernel/adam_v" string_val: "bert/encoder/layer_9/output/layer_normalization_20/beta" string_val: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" string_val: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" string_val: "bert/encoder/layer_9/output/layer_normalization_20/gamma" string_val: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" string_val: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" string_val: "bert/pooler/dense/bias" string_val: "bert/pooler/dense/bias/adam_m" string_val: "bert/pooler/dense/bias/adam_v" string_val: "bert/pooler/dense/kernel" string_val: "bert/pooler/dense/kernel/adam_m" string_val: "bert/pooler/dense/kernel/adam_v" string_val: "global_step" string_val: "output_bias" string_val: "output_bias/adam_m" string_val: "output_bias/adam_v" string_val: "output_weights" string_val: "output_weights/adam_m" string_val: "output_weights/adam_v" } } } } node { name: "save/RestoreV2/shape_and_slices" op: "Const" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { dim { size: 604 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 604 } } string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" string_val: "" } } } } node { name: "save/RestoreV2" op: "RestoreV2" input: "save/Const" input: "save/RestoreV2/tensor_names" input: "save/RestoreV2/shape_and_slices" device: "/device:CPU:0" attr { key: "_output_shapes" value { list { shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } shape { unknown_rank: true } } } } attr { key: "dtypes" value { list { type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_INT64 type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT type: DT_FLOAT } } } } node { name: "save/Identity_1" op: "Identity" input: "save/RestoreV2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/beta" input: "save/Identity_1" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_2" op: "Identity" input: "save/RestoreV2:1" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_1" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/beta/adam_m" input: "save/Identity_2" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_3" op: "Identity" input: "save/RestoreV2:2" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_2" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/beta/adam_v" input: "save/Identity_3" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_4" op: "Identity" input: "save/RestoreV2:3" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_3" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/gamma" input: "save/Identity_4" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_5" op: "Identity" input: "save/RestoreV2:4" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_4" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/gamma/adam_m" input: "save/Identity_5" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_6" op: "Identity" input: "save/RestoreV2:5" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_5" op: "AssignVariableOp" input: "bert/embeddings/layer_normalization/gamma/adam_v" input: "save/Identity_6" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_7" op: "Identity" input: "save/RestoreV2:6" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_6" op: "AssignVariableOp" input: "bert/embeddings/position_embeddings" input: "save/Identity_7" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_8" op: "Identity" input: "save/RestoreV2:7" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_7" op: "AssignVariableOp" input: "bert/embeddings/position_embeddings/adam_m" input: "save/Identity_8" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_9" op: "Identity" input: "save/RestoreV2:8" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_8" op: "AssignVariableOp" input: "bert/embeddings/position_embeddings/adam_v" input: "save/Identity_9" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_10" op: "Identity" input: "save/RestoreV2:9" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_9" op: "AssignVariableOp" input: "bert/embeddings/token_type_embeddings" input: "save/Identity_10" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_11" op: "Identity" input: "save/RestoreV2:10" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_10" op: "AssignVariableOp" input: "bert/embeddings/token_type_embeddings/adam_m" input: "save/Identity_11" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_12" op: "Identity" input: "save/RestoreV2:11" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_11" op: "AssignVariableOp" input: "bert/embeddings/token_type_embeddings/adam_v" input: "save/Identity_12" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_13" op: "Identity" input: "save/RestoreV2:12" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_12" op: "AssignVariableOp" input: "bert/embeddings/word_embeddings" input: "save/Identity_13" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_14" op: "Identity" input: "save/RestoreV2:13" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_13" op: "AssignVariableOp" input: "bert/embeddings/word_embeddings/adam_m" input: "save/Identity_14" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_15" op: "Identity" input: "save/RestoreV2:14" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_14" op: "AssignVariableOp" input: "bert/embeddings/word_embeddings/adam_v" input: "save/Identity_15" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_16" op: "Identity" input: "save/RestoreV2:15" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_15" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias" input: "save/Identity_16" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_17" op: "Identity" input: "save/RestoreV2:16" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_16" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_m" input: "save/Identity_17" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_18" op: "Identity" input: "save/RestoreV2:17" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_17" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/bias/adam_v" input: "save/Identity_18" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_19" op: "Identity" input: "save/RestoreV2:18" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_18" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel" input: "save/Identity_19" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_20" op: "Identity" input: "save/RestoreV2:19" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_19" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_m" input: "save/Identity_20" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_21" op: "Identity" input: "save/RestoreV2:20" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_20" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/dense/kernel/adam_v" input: "save/Identity_21" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_22" op: "Identity" input: "save/RestoreV2:21" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_21" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta" input: "save/Identity_22" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_23" op: "Identity" input: "save/RestoreV2:22" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_22" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_m" input: "save/Identity_23" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_24" op: "Identity" input: "save/RestoreV2:23" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_23" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/beta/adam_v" input: "save/Identity_24" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_25" op: "Identity" input: "save/RestoreV2:24" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_24" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma" input: "save/Identity_25" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_26" op: "Identity" input: "save/RestoreV2:25" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_25" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_m" input: "save/Identity_26" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_27" op: "Identity" input: "save/RestoreV2:26" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_26" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/output/layer_normalization_1/gamma/adam_v" input: "save/Identity_27" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_28" op: "Identity" input: "save/RestoreV2:27" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_27" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias" input: "save/Identity_28" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_29" op: "Identity" input: "save/RestoreV2:28" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_28" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_m" input: "save/Identity_29" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_30" op: "Identity" input: "save/RestoreV2:29" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_29" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/bias/adam_v" input: "save/Identity_30" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_31" op: "Identity" input: "save/RestoreV2:30" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_30" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel" input: "save/Identity_31" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_32" op: "Identity" input: "save/RestoreV2:31" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_31" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_m" input: "save/Identity_32" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_33" op: "Identity" input: "save/RestoreV2:32" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_32" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/key/kernel/adam_v" input: "save/Identity_33" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_34" op: "Identity" input: "save/RestoreV2:33" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_33" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias" input: "save/Identity_34" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_35" op: "Identity" input: "save/RestoreV2:34" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_34" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_m" input: "save/Identity_35" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_36" op: "Identity" input: "save/RestoreV2:35" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_35" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/bias/adam_v" input: "save/Identity_36" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_37" op: "Identity" input: "save/RestoreV2:36" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_36" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel" input: "save/Identity_37" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_38" op: "Identity" input: "save/RestoreV2:37" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_37" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_m" input: "save/Identity_38" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_39" op: "Identity" input: "save/RestoreV2:38" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_38" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/query/kernel/adam_v" input: "save/Identity_39" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_40" op: "Identity" input: "save/RestoreV2:39" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_39" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias" input: "save/Identity_40" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_41" op: "Identity" input: "save/RestoreV2:40" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_40" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_m" input: "save/Identity_41" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_42" op: "Identity" input: "save/RestoreV2:41" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_41" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/bias/adam_v" input: "save/Identity_42" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_43" op: "Identity" input: "save/RestoreV2:42" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_42" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel" input: "save/Identity_43" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_44" op: "Identity" input: "save/RestoreV2:43" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_43" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_m" input: "save/Identity_44" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_45" op: "Identity" input: "save/RestoreV2:44" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_44" op: "AssignVariableOp" input: "bert/encoder/layer_0/attention/self/value/kernel/adam_v" input: "save/Identity_45" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_46" op: "Identity" input: "save/RestoreV2:45" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_45" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias" input: "save/Identity_46" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_47" op: "Identity" input: "save/RestoreV2:46" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_46" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_m" input: "save/Identity_47" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_48" op: "Identity" input: "save/RestoreV2:47" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_47" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/bias/adam_v" input: "save/Identity_48" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_49" op: "Identity" input: "save/RestoreV2:48" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_48" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel" input: "save/Identity_49" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_50" op: "Identity" input: "save/RestoreV2:49" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_49" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_m" input: "save/Identity_50" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_51" op: "Identity" input: "save/RestoreV2:50" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_50" op: "AssignVariableOp" input: "bert/encoder/layer_0/intermediate/dense/kernel/adam_v" input: "save/Identity_51" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_52" op: "Identity" input: "save/RestoreV2:51" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_51" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/bias" input: "save/Identity_52" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_53" op: "Identity" input: "save/RestoreV2:52" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_52" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/bias/adam_m" input: "save/Identity_53" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_54" op: "Identity" input: "save/RestoreV2:53" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_53" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/bias/adam_v" input: "save/Identity_54" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_55" op: "Identity" input: "save/RestoreV2:54" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_54" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/kernel" input: "save/Identity_55" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_56" op: "Identity" input: "save/RestoreV2:55" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_55" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_m" input: "save/Identity_56" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_57" op: "Identity" input: "save/RestoreV2:56" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_56" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/dense/kernel/adam_v" input: "save/Identity_57" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_58" op: "Identity" input: "save/RestoreV2:57" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_57" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta" input: "save/Identity_58" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_59" op: "Identity" input: "save/RestoreV2:58" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_58" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_m" input: "save/Identity_59" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_60" op: "Identity" input: "save/RestoreV2:59" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_59" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/beta/adam_v" input: "save/Identity_60" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_61" op: "Identity" input: "save/RestoreV2:60" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_60" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma" input: "save/Identity_61" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_62" op: "Identity" input: "save/RestoreV2:61" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_61" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_m" input: "save/Identity_62" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_63" op: "Identity" input: "save/RestoreV2:62" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_62" op: "AssignVariableOp" input: "bert/encoder/layer_0/output/layer_normalization_2/gamma/adam_v" input: "save/Identity_63" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_64" op: "Identity" input: "save/RestoreV2:63" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_63" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias" input: "save/Identity_64" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_65" op: "Identity" input: "save/RestoreV2:64" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_64" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_m" input: "save/Identity_65" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_66" op: "Identity" input: "save/RestoreV2:65" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_65" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/bias/adam_v" input: "save/Identity_66" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_67" op: "Identity" input: "save/RestoreV2:66" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_66" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel" input: "save/Identity_67" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_68" op: "Identity" input: "save/RestoreV2:67" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_67" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_m" input: "save/Identity_68" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_69" op: "Identity" input: "save/RestoreV2:68" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_68" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/dense/kernel/adam_v" input: "save/Identity_69" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_70" op: "Identity" input: "save/RestoreV2:69" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_69" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta" input: "save/Identity_70" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_71" op: "Identity" input: "save/RestoreV2:70" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_70" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_m" input: "save/Identity_71" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_72" op: "Identity" input: "save/RestoreV2:71" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_71" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/beta/adam_v" input: "save/Identity_72" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_73" op: "Identity" input: "save/RestoreV2:72" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_72" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma" input: "save/Identity_73" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_74" op: "Identity" input: "save/RestoreV2:73" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_73" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_m" input: "save/Identity_74" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_75" op: "Identity" input: "save/RestoreV2:74" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_74" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/output/layer_normalization_3/gamma/adam_v" input: "save/Identity_75" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_76" op: "Identity" input: "save/RestoreV2:75" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_75" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias" input: "save/Identity_76" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_77" op: "Identity" input: "save/RestoreV2:76" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_76" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_m" input: "save/Identity_77" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_78" op: "Identity" input: "save/RestoreV2:77" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_77" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/bias/adam_v" input: "save/Identity_78" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_79" op: "Identity" input: "save/RestoreV2:78" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_78" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel" input: "save/Identity_79" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_80" op: "Identity" input: "save/RestoreV2:79" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_79" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_m" input: "save/Identity_80" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_81" op: "Identity" input: "save/RestoreV2:80" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_80" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/key/kernel/adam_v" input: "save/Identity_81" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_82" op: "Identity" input: "save/RestoreV2:81" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_81" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias" input: "save/Identity_82" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_83" op: "Identity" input: "save/RestoreV2:82" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_82" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_m" input: "save/Identity_83" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_84" op: "Identity" input: "save/RestoreV2:83" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_83" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/bias/adam_v" input: "save/Identity_84" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_85" op: "Identity" input: "save/RestoreV2:84" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_84" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel" input: "save/Identity_85" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_86" op: "Identity" input: "save/RestoreV2:85" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_85" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_m" input: "save/Identity_86" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_87" op: "Identity" input: "save/RestoreV2:86" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_86" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/query/kernel/adam_v" input: "save/Identity_87" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_88" op: "Identity" input: "save/RestoreV2:87" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_87" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias" input: "save/Identity_88" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_89" op: "Identity" input: "save/RestoreV2:88" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_88" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_m" input: "save/Identity_89" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_90" op: "Identity" input: "save/RestoreV2:89" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_89" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/bias/adam_v" input: "save/Identity_90" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_91" op: "Identity" input: "save/RestoreV2:90" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_90" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel" input: "save/Identity_91" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_92" op: "Identity" input: "save/RestoreV2:91" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_91" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_m" input: "save/Identity_92" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_93" op: "Identity" input: "save/RestoreV2:92" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_92" op: "AssignVariableOp" input: "bert/encoder/layer_1/attention/self/value/kernel/adam_v" input: "save/Identity_93" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_94" op: "Identity" input: "save/RestoreV2:93" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_93" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias" input: "save/Identity_94" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_95" op: "Identity" input: "save/RestoreV2:94" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_94" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_m" input: "save/Identity_95" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_96" op: "Identity" input: "save/RestoreV2:95" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_95" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/bias/adam_v" input: "save/Identity_96" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_97" op: "Identity" input: "save/RestoreV2:96" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_96" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel" input: "save/Identity_97" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_98" op: "Identity" input: "save/RestoreV2:97" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_97" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_m" input: "save/Identity_98" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_99" op: "Identity" input: "save/RestoreV2:98" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_98" op: "AssignVariableOp" input: "bert/encoder/layer_1/intermediate/dense/kernel/adam_v" input: "save/Identity_99" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_100" op: "Identity" input: "save/RestoreV2:99" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_99" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/bias" input: "save/Identity_100" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_101" op: "Identity" input: "save/RestoreV2:100" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_100" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/bias/adam_m" input: "save/Identity_101" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_102" op: "Identity" input: "save/RestoreV2:101" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_101" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/bias/adam_v" input: "save/Identity_102" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_103" op: "Identity" input: "save/RestoreV2:102" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_102" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/kernel" input: "save/Identity_103" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_104" op: "Identity" input: "save/RestoreV2:103" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_103" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_m" input: "save/Identity_104" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_105" op: "Identity" input: "save/RestoreV2:104" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_104" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/dense/kernel/adam_v" input: "save/Identity_105" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_106" op: "Identity" input: "save/RestoreV2:105" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_105" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta" input: "save/Identity_106" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_107" op: "Identity" input: "save/RestoreV2:106" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_106" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_m" input: "save/Identity_107" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_108" op: "Identity" input: "save/RestoreV2:107" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_107" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/beta/adam_v" input: "save/Identity_108" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_109" op: "Identity" input: "save/RestoreV2:108" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_108" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma" input: "save/Identity_109" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_110" op: "Identity" input: "save/RestoreV2:109" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_109" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_m" input: "save/Identity_110" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_111" op: "Identity" input: "save/RestoreV2:110" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_110" op: "AssignVariableOp" input: "bert/encoder/layer_1/output/layer_normalization_4/gamma/adam_v" input: "save/Identity_111" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_112" op: "Identity" input: "save/RestoreV2:111" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_111" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias" input: "save/Identity_112" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_113" op: "Identity" input: "save/RestoreV2:112" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_112" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_m" input: "save/Identity_113" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_114" op: "Identity" input: "save/RestoreV2:113" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_113" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/bias/adam_v" input: "save/Identity_114" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_115" op: "Identity" input: "save/RestoreV2:114" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_114" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel" input: "save/Identity_115" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_116" op: "Identity" input: "save/RestoreV2:115" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_115" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_m" input: "save/Identity_116" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_117" op: "Identity" input: "save/RestoreV2:116" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_116" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/dense/kernel/adam_v" input: "save/Identity_117" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_118" op: "Identity" input: "save/RestoreV2:117" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_117" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta" input: "save/Identity_118" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_119" op: "Identity" input: "save/RestoreV2:118" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_118" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_m" input: "save/Identity_119" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_120" op: "Identity" input: "save/RestoreV2:119" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_119" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/beta/adam_v" input: "save/Identity_120" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_121" op: "Identity" input: "save/RestoreV2:120" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_120" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma" input: "save/Identity_121" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_122" op: "Identity" input: "save/RestoreV2:121" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_121" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_m" input: "save/Identity_122" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_123" op: "Identity" input: "save/RestoreV2:122" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_122" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/output/layer_normalization_21/gamma/adam_v" input: "save/Identity_123" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_124" op: "Identity" input: "save/RestoreV2:123" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_123" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias" input: "save/Identity_124" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_125" op: "Identity" input: "save/RestoreV2:124" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_124" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_m" input: "save/Identity_125" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_126" op: "Identity" input: "save/RestoreV2:125" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_125" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/bias/adam_v" input: "save/Identity_126" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_127" op: "Identity" input: "save/RestoreV2:126" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_126" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel" input: "save/Identity_127" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_128" op: "Identity" input: "save/RestoreV2:127" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_127" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_m" input: "save/Identity_128" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_129" op: "Identity" input: "save/RestoreV2:128" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_128" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/key/kernel/adam_v" input: "save/Identity_129" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_130" op: "Identity" input: "save/RestoreV2:129" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_129" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias" input: "save/Identity_130" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_131" op: "Identity" input: "save/RestoreV2:130" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_130" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_m" input: "save/Identity_131" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_132" op: "Identity" input: "save/RestoreV2:131" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_131" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/bias/adam_v" input: "save/Identity_132" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_133" op: "Identity" input: "save/RestoreV2:132" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_132" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel" input: "save/Identity_133" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_134" op: "Identity" input: "save/RestoreV2:133" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_133" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_m" input: "save/Identity_134" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_135" op: "Identity" input: "save/RestoreV2:134" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_134" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/query/kernel/adam_v" input: "save/Identity_135" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_136" op: "Identity" input: "save/RestoreV2:135" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_135" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias" input: "save/Identity_136" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_137" op: "Identity" input: "save/RestoreV2:136" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_136" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_m" input: "save/Identity_137" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_138" op: "Identity" input: "save/RestoreV2:137" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_137" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/bias/adam_v" input: "save/Identity_138" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_139" op: "Identity" input: "save/RestoreV2:138" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_138" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel" input: "save/Identity_139" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_140" op: "Identity" input: "save/RestoreV2:139" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_139" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_m" input: "save/Identity_140" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_141" op: "Identity" input: "save/RestoreV2:140" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_140" op: "AssignVariableOp" input: "bert/encoder/layer_10/attention/self/value/kernel/adam_v" input: "save/Identity_141" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_142" op: "Identity" input: "save/RestoreV2:141" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_141" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias" input: "save/Identity_142" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_143" op: "Identity" input: "save/RestoreV2:142" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_142" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_m" input: "save/Identity_143" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_144" op: "Identity" input: "save/RestoreV2:143" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_143" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/bias/adam_v" input: "save/Identity_144" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_145" op: "Identity" input: "save/RestoreV2:144" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_144" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel" input: "save/Identity_145" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_146" op: "Identity" input: "save/RestoreV2:145" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_145" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_m" input: "save/Identity_146" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_147" op: "Identity" input: "save/RestoreV2:146" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_146" op: "AssignVariableOp" input: "bert/encoder/layer_10/intermediate/dense/kernel/adam_v" input: "save/Identity_147" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_148" op: "Identity" input: "save/RestoreV2:147" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_147" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/bias" input: "save/Identity_148" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_149" op: "Identity" input: "save/RestoreV2:148" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_148" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/bias/adam_m" input: "save/Identity_149" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_150" op: "Identity" input: "save/RestoreV2:149" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_149" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/bias/adam_v" input: "save/Identity_150" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_151" op: "Identity" input: "save/RestoreV2:150" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_150" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/kernel" input: "save/Identity_151" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_152" op: "Identity" input: "save/RestoreV2:151" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_151" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_m" input: "save/Identity_152" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_153" op: "Identity" input: "save/RestoreV2:152" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_152" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/dense/kernel/adam_v" input: "save/Identity_153" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_154" op: "Identity" input: "save/RestoreV2:153" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_153" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta" input: "save/Identity_154" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_155" op: "Identity" input: "save/RestoreV2:154" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_154" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_m" input: "save/Identity_155" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_156" op: "Identity" input: "save/RestoreV2:155" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_155" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/beta/adam_v" input: "save/Identity_156" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_157" op: "Identity" input: "save/RestoreV2:156" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_156" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma" input: "save/Identity_157" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_158" op: "Identity" input: "save/RestoreV2:157" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_157" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_m" input: "save/Identity_158" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_159" op: "Identity" input: "save/RestoreV2:158" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_158" op: "AssignVariableOp" input: "bert/encoder/layer_10/output/layer_normalization_22/gamma/adam_v" input: "save/Identity_159" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_160" op: "Identity" input: "save/RestoreV2:159" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_159" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias" input: "save/Identity_160" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_161" op: "Identity" input: "save/RestoreV2:160" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_160" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_m" input: "save/Identity_161" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_162" op: "Identity" input: "save/RestoreV2:161" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_161" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/bias/adam_v" input: "save/Identity_162" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_163" op: "Identity" input: "save/RestoreV2:162" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_162" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel" input: "save/Identity_163" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_164" op: "Identity" input: "save/RestoreV2:163" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_163" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_m" input: "save/Identity_164" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_165" op: "Identity" input: "save/RestoreV2:164" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_164" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/dense/kernel/adam_v" input: "save/Identity_165" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_166" op: "Identity" input: "save/RestoreV2:165" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_165" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta" input: "save/Identity_166" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_167" op: "Identity" input: "save/RestoreV2:166" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_166" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_m" input: "save/Identity_167" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_168" op: "Identity" input: "save/RestoreV2:167" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_167" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/beta/adam_v" input: "save/Identity_168" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_169" op: "Identity" input: "save/RestoreV2:168" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_168" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma" input: "save/Identity_169" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_170" op: "Identity" input: "save/RestoreV2:169" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_169" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_m" input: "save/Identity_170" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_171" op: "Identity" input: "save/RestoreV2:170" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_170" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/output/layer_normalization_23/gamma/adam_v" input: "save/Identity_171" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_172" op: "Identity" input: "save/RestoreV2:171" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_171" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias" input: "save/Identity_172" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_173" op: "Identity" input: "save/RestoreV2:172" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_172" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_m" input: "save/Identity_173" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_174" op: "Identity" input: "save/RestoreV2:173" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_173" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/bias/adam_v" input: "save/Identity_174" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_175" op: "Identity" input: "save/RestoreV2:174" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_174" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel" input: "save/Identity_175" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_176" op: "Identity" input: "save/RestoreV2:175" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_175" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_m" input: "save/Identity_176" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_177" op: "Identity" input: "save/RestoreV2:176" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_176" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/key/kernel/adam_v" input: "save/Identity_177" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_178" op: "Identity" input: "save/RestoreV2:177" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_177" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias" input: "save/Identity_178" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_179" op: "Identity" input: "save/RestoreV2:178" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_178" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_m" input: "save/Identity_179" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_180" op: "Identity" input: "save/RestoreV2:179" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_179" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/bias/adam_v" input: "save/Identity_180" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_181" op: "Identity" input: "save/RestoreV2:180" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_180" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel" input: "save/Identity_181" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_182" op: "Identity" input: "save/RestoreV2:181" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_181" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_m" input: "save/Identity_182" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_183" op: "Identity" input: "save/RestoreV2:182" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_182" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/query/kernel/adam_v" input: "save/Identity_183" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_184" op: "Identity" input: "save/RestoreV2:183" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_183" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias" input: "save/Identity_184" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_185" op: "Identity" input: "save/RestoreV2:184" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_184" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_m" input: "save/Identity_185" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_186" op: "Identity" input: "save/RestoreV2:185" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_185" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/bias/adam_v" input: "save/Identity_186" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_187" op: "Identity" input: "save/RestoreV2:186" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_186" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel" input: "save/Identity_187" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_188" op: "Identity" input: "save/RestoreV2:187" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_187" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_m" input: "save/Identity_188" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_189" op: "Identity" input: "save/RestoreV2:188" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_188" op: "AssignVariableOp" input: "bert/encoder/layer_11/attention/self/value/kernel/adam_v" input: "save/Identity_189" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_190" op: "Identity" input: "save/RestoreV2:189" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_189" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias" input: "save/Identity_190" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_191" op: "Identity" input: "save/RestoreV2:190" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_190" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_m" input: "save/Identity_191" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_192" op: "Identity" input: "save/RestoreV2:191" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_191" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/bias/adam_v" input: "save/Identity_192" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_193" op: "Identity" input: "save/RestoreV2:192" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_192" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel" input: "save/Identity_193" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_194" op: "Identity" input: "save/RestoreV2:193" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_193" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_m" input: "save/Identity_194" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_195" op: "Identity" input: "save/RestoreV2:194" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_194" op: "AssignVariableOp" input: "bert/encoder/layer_11/intermediate/dense/kernel/adam_v" input: "save/Identity_195" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_196" op: "Identity" input: "save/RestoreV2:195" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_195" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/bias" input: "save/Identity_196" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_197" op: "Identity" input: "save/RestoreV2:196" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_196" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/bias/adam_m" input: "save/Identity_197" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_198" op: "Identity" input: "save/RestoreV2:197" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_197" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/bias/adam_v" input: "save/Identity_198" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_199" op: "Identity" input: "save/RestoreV2:198" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_198" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/kernel" input: "save/Identity_199" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_200" op: "Identity" input: "save/RestoreV2:199" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_199" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_m" input: "save/Identity_200" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_201" op: "Identity" input: "save/RestoreV2:200" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_200" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/dense/kernel/adam_v" input: "save/Identity_201" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_202" op: "Identity" input: "save/RestoreV2:201" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_201" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta" input: "save/Identity_202" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_203" op: "Identity" input: "save/RestoreV2:202" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_202" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_m" input: "save/Identity_203" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_204" op: "Identity" input: "save/RestoreV2:203" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_203" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/beta/adam_v" input: "save/Identity_204" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_205" op: "Identity" input: "save/RestoreV2:204" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_204" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma" input: "save/Identity_205" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_206" op: "Identity" input: "save/RestoreV2:205" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_205" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_m" input: "save/Identity_206" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_207" op: "Identity" input: "save/RestoreV2:206" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_206" op: "AssignVariableOp" input: "bert/encoder/layer_11/output/layer_normalization_24/gamma/adam_v" input: "save/Identity_207" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_208" op: "Identity" input: "save/RestoreV2:207" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_207" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias" input: "save/Identity_208" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_209" op: "Identity" input: "save/RestoreV2:208" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_208" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_m" input: "save/Identity_209" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_210" op: "Identity" input: "save/RestoreV2:209" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_209" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/bias/adam_v" input: "save/Identity_210" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_211" op: "Identity" input: "save/RestoreV2:210" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_210" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel" input: "save/Identity_211" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_212" op: "Identity" input: "save/RestoreV2:211" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_211" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_m" input: "save/Identity_212" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_213" op: "Identity" input: "save/RestoreV2:212" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_212" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/dense/kernel/adam_v" input: "save/Identity_213" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_214" op: "Identity" input: "save/RestoreV2:213" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_213" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta" input: "save/Identity_214" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_215" op: "Identity" input: "save/RestoreV2:214" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_214" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_m" input: "save/Identity_215" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_216" op: "Identity" input: "save/RestoreV2:215" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_215" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/beta/adam_v" input: "save/Identity_216" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_217" op: "Identity" input: "save/RestoreV2:216" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_216" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma" input: "save/Identity_217" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_218" op: "Identity" input: "save/RestoreV2:217" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_217" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_m" input: "save/Identity_218" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_219" op: "Identity" input: "save/RestoreV2:218" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_218" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/output/layer_normalization_5/gamma/adam_v" input: "save/Identity_219" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_220" op: "Identity" input: "save/RestoreV2:219" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_219" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias" input: "save/Identity_220" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_221" op: "Identity" input: "save/RestoreV2:220" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_220" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_m" input: "save/Identity_221" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_222" op: "Identity" input: "save/RestoreV2:221" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_221" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/bias/adam_v" input: "save/Identity_222" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_223" op: "Identity" input: "save/RestoreV2:222" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_222" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel" input: "save/Identity_223" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_224" op: "Identity" input: "save/RestoreV2:223" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_223" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_m" input: "save/Identity_224" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_225" op: "Identity" input: "save/RestoreV2:224" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_224" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/key/kernel/adam_v" input: "save/Identity_225" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_226" op: "Identity" input: "save/RestoreV2:225" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_225" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias" input: "save/Identity_226" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_227" op: "Identity" input: "save/RestoreV2:226" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_226" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_m" input: "save/Identity_227" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_228" op: "Identity" input: "save/RestoreV2:227" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_227" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/bias/adam_v" input: "save/Identity_228" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_229" op: "Identity" input: "save/RestoreV2:228" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_228" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel" input: "save/Identity_229" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_230" op: "Identity" input: "save/RestoreV2:229" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_229" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_m" input: "save/Identity_230" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_231" op: "Identity" input: "save/RestoreV2:230" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_230" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/query/kernel/adam_v" input: "save/Identity_231" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_232" op: "Identity" input: "save/RestoreV2:231" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_231" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias" input: "save/Identity_232" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_233" op: "Identity" input: "save/RestoreV2:232" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_232" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_m" input: "save/Identity_233" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_234" op: "Identity" input: "save/RestoreV2:233" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_233" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/bias/adam_v" input: "save/Identity_234" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_235" op: "Identity" input: "save/RestoreV2:234" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_234" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel" input: "save/Identity_235" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_236" op: "Identity" input: "save/RestoreV2:235" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_235" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_m" input: "save/Identity_236" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_237" op: "Identity" input: "save/RestoreV2:236" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_236" op: "AssignVariableOp" input: "bert/encoder/layer_2/attention/self/value/kernel/adam_v" input: "save/Identity_237" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_238" op: "Identity" input: "save/RestoreV2:237" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_237" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias" input: "save/Identity_238" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_239" op: "Identity" input: "save/RestoreV2:238" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_238" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_m" input: "save/Identity_239" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_240" op: "Identity" input: "save/RestoreV2:239" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_239" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/bias/adam_v" input: "save/Identity_240" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_241" op: "Identity" input: "save/RestoreV2:240" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_240" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel" input: "save/Identity_241" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_242" op: "Identity" input: "save/RestoreV2:241" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_241" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_m" input: "save/Identity_242" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_243" op: "Identity" input: "save/RestoreV2:242" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_242" op: "AssignVariableOp" input: "bert/encoder/layer_2/intermediate/dense/kernel/adam_v" input: "save/Identity_243" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_244" op: "Identity" input: "save/RestoreV2:243" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_243" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/bias" input: "save/Identity_244" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_245" op: "Identity" input: "save/RestoreV2:244" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_244" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/bias/adam_m" input: "save/Identity_245" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_246" op: "Identity" input: "save/RestoreV2:245" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_245" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/bias/adam_v" input: "save/Identity_246" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_247" op: "Identity" input: "save/RestoreV2:246" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_246" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/kernel" input: "save/Identity_247" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_248" op: "Identity" input: "save/RestoreV2:247" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_247" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_m" input: "save/Identity_248" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_249" op: "Identity" input: "save/RestoreV2:248" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_248" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/dense/kernel/adam_v" input: "save/Identity_249" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_250" op: "Identity" input: "save/RestoreV2:249" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_249" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta" input: "save/Identity_250" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_251" op: "Identity" input: "save/RestoreV2:250" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_250" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_m" input: "save/Identity_251" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_252" op: "Identity" input: "save/RestoreV2:251" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_251" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/beta/adam_v" input: "save/Identity_252" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_253" op: "Identity" input: "save/RestoreV2:252" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_252" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma" input: "save/Identity_253" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_254" op: "Identity" input: "save/RestoreV2:253" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_253" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_m" input: "save/Identity_254" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_255" op: "Identity" input: "save/RestoreV2:254" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_254" op: "AssignVariableOp" input: "bert/encoder/layer_2/output/layer_normalization_6/gamma/adam_v" input: "save/Identity_255" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_256" op: "Identity" input: "save/RestoreV2:255" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_255" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias" input: "save/Identity_256" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_257" op: "Identity" input: "save/RestoreV2:256" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_256" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_m" input: "save/Identity_257" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_258" op: "Identity" input: "save/RestoreV2:257" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_257" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/bias/adam_v" input: "save/Identity_258" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_259" op: "Identity" input: "save/RestoreV2:258" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_258" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel" input: "save/Identity_259" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_260" op: "Identity" input: "save/RestoreV2:259" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_259" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_m" input: "save/Identity_260" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_261" op: "Identity" input: "save/RestoreV2:260" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_260" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/dense/kernel/adam_v" input: "save/Identity_261" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_262" op: "Identity" input: "save/RestoreV2:261" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_261" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta" input: "save/Identity_262" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_263" op: "Identity" input: "save/RestoreV2:262" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_262" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_m" input: "save/Identity_263" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_264" op: "Identity" input: "save/RestoreV2:263" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_263" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/beta/adam_v" input: "save/Identity_264" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_265" op: "Identity" input: "save/RestoreV2:264" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_264" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma" input: "save/Identity_265" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_266" op: "Identity" input: "save/RestoreV2:265" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_265" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_m" input: "save/Identity_266" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_267" op: "Identity" input: "save/RestoreV2:266" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_266" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/output/layer_normalization_7/gamma/adam_v" input: "save/Identity_267" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_268" op: "Identity" input: "save/RestoreV2:267" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_267" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias" input: "save/Identity_268" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_269" op: "Identity" input: "save/RestoreV2:268" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_268" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_m" input: "save/Identity_269" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_270" op: "Identity" input: "save/RestoreV2:269" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_269" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/bias/adam_v" input: "save/Identity_270" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_271" op: "Identity" input: "save/RestoreV2:270" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_270" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel" input: "save/Identity_271" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_272" op: "Identity" input: "save/RestoreV2:271" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_271" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_m" input: "save/Identity_272" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_273" op: "Identity" input: "save/RestoreV2:272" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_272" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/key/kernel/adam_v" input: "save/Identity_273" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_274" op: "Identity" input: "save/RestoreV2:273" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_273" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias" input: "save/Identity_274" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_275" op: "Identity" input: "save/RestoreV2:274" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_274" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_m" input: "save/Identity_275" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_276" op: "Identity" input: "save/RestoreV2:275" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_275" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/bias/adam_v" input: "save/Identity_276" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_277" op: "Identity" input: "save/RestoreV2:276" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_276" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel" input: "save/Identity_277" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_278" op: "Identity" input: "save/RestoreV2:277" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_277" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_m" input: "save/Identity_278" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_279" op: "Identity" input: "save/RestoreV2:278" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_278" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/query/kernel/adam_v" input: "save/Identity_279" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_280" op: "Identity" input: "save/RestoreV2:279" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_279" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias" input: "save/Identity_280" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_281" op: "Identity" input: "save/RestoreV2:280" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_280" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_m" input: "save/Identity_281" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_282" op: "Identity" input: "save/RestoreV2:281" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_281" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/bias/adam_v" input: "save/Identity_282" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_283" op: "Identity" input: "save/RestoreV2:282" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_282" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel" input: "save/Identity_283" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_284" op: "Identity" input: "save/RestoreV2:283" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_283" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_m" input: "save/Identity_284" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_285" op: "Identity" input: "save/RestoreV2:284" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_284" op: "AssignVariableOp" input: "bert/encoder/layer_3/attention/self/value/kernel/adam_v" input: "save/Identity_285" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_286" op: "Identity" input: "save/RestoreV2:285" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_285" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias" input: "save/Identity_286" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_287" op: "Identity" input: "save/RestoreV2:286" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_286" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_m" input: "save/Identity_287" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_288" op: "Identity" input: "save/RestoreV2:287" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_287" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/bias/adam_v" input: "save/Identity_288" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_289" op: "Identity" input: "save/RestoreV2:288" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_288" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel" input: "save/Identity_289" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_290" op: "Identity" input: "save/RestoreV2:289" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_289" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_m" input: "save/Identity_290" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_291" op: "Identity" input: "save/RestoreV2:290" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_290" op: "AssignVariableOp" input: "bert/encoder/layer_3/intermediate/dense/kernel/adam_v" input: "save/Identity_291" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_292" op: "Identity" input: "save/RestoreV2:291" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_291" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/bias" input: "save/Identity_292" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_293" op: "Identity" input: "save/RestoreV2:292" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_292" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/bias/adam_m" input: "save/Identity_293" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_294" op: "Identity" input: "save/RestoreV2:293" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_293" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/bias/adam_v" input: "save/Identity_294" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_295" op: "Identity" input: "save/RestoreV2:294" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_294" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/kernel" input: "save/Identity_295" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_296" op: "Identity" input: "save/RestoreV2:295" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_295" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_m" input: "save/Identity_296" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_297" op: "Identity" input: "save/RestoreV2:296" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_296" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/dense/kernel/adam_v" input: "save/Identity_297" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_298" op: "Identity" input: "save/RestoreV2:297" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_297" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta" input: "save/Identity_298" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_299" op: "Identity" input: "save/RestoreV2:298" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_298" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_m" input: "save/Identity_299" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_300" op: "Identity" input: "save/RestoreV2:299" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_299" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/beta/adam_v" input: "save/Identity_300" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_301" op: "Identity" input: "save/RestoreV2:300" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_300" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma" input: "save/Identity_301" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_302" op: "Identity" input: "save/RestoreV2:301" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_301" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_m" input: "save/Identity_302" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_303" op: "Identity" input: "save/RestoreV2:302" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_302" op: "AssignVariableOp" input: "bert/encoder/layer_3/output/layer_normalization_8/gamma/adam_v" input: "save/Identity_303" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_304" op: "Identity" input: "save/RestoreV2:303" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_303" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias" input: "save/Identity_304" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_305" op: "Identity" input: "save/RestoreV2:304" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_304" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_m" input: "save/Identity_305" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_306" op: "Identity" input: "save/RestoreV2:305" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_305" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/bias/adam_v" input: "save/Identity_306" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_307" op: "Identity" input: "save/RestoreV2:306" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_306" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel" input: "save/Identity_307" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_308" op: "Identity" input: "save/RestoreV2:307" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_307" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_m" input: "save/Identity_308" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_309" op: "Identity" input: "save/RestoreV2:308" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_308" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/dense/kernel/adam_v" input: "save/Identity_309" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_310" op: "Identity" input: "save/RestoreV2:309" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_309" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta" input: "save/Identity_310" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_311" op: "Identity" input: "save/RestoreV2:310" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_310" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_m" input: "save/Identity_311" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_312" op: "Identity" input: "save/RestoreV2:311" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_311" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/beta/adam_v" input: "save/Identity_312" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_313" op: "Identity" input: "save/RestoreV2:312" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_312" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma" input: "save/Identity_313" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_314" op: "Identity" input: "save/RestoreV2:313" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_313" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_m" input: "save/Identity_314" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_315" op: "Identity" input: "save/RestoreV2:314" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_314" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/output/layer_normalization_9/gamma/adam_v" input: "save/Identity_315" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_316" op: "Identity" input: "save/RestoreV2:315" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_315" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias" input: "save/Identity_316" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_317" op: "Identity" input: "save/RestoreV2:316" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_316" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_m" input: "save/Identity_317" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_318" op: "Identity" input: "save/RestoreV2:317" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_317" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/bias/adam_v" input: "save/Identity_318" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_319" op: "Identity" input: "save/RestoreV2:318" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_318" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel" input: "save/Identity_319" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_320" op: "Identity" input: "save/RestoreV2:319" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_319" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_m" input: "save/Identity_320" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_321" op: "Identity" input: "save/RestoreV2:320" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_320" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/key/kernel/adam_v" input: "save/Identity_321" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_322" op: "Identity" input: "save/RestoreV2:321" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_321" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias" input: "save/Identity_322" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_323" op: "Identity" input: "save/RestoreV2:322" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_322" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_m" input: "save/Identity_323" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_324" op: "Identity" input: "save/RestoreV2:323" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_323" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/bias/adam_v" input: "save/Identity_324" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_325" op: "Identity" input: "save/RestoreV2:324" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_324" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel" input: "save/Identity_325" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_326" op: "Identity" input: "save/RestoreV2:325" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_325" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_m" input: "save/Identity_326" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_327" op: "Identity" input: "save/RestoreV2:326" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_326" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/query/kernel/adam_v" input: "save/Identity_327" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_328" op: "Identity" input: "save/RestoreV2:327" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_327" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias" input: "save/Identity_328" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_329" op: "Identity" input: "save/RestoreV2:328" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_328" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_m" input: "save/Identity_329" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_330" op: "Identity" input: "save/RestoreV2:329" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_329" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/bias/adam_v" input: "save/Identity_330" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_331" op: "Identity" input: "save/RestoreV2:330" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_330" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel" input: "save/Identity_331" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_332" op: "Identity" input: "save/RestoreV2:331" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_331" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_m" input: "save/Identity_332" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_333" op: "Identity" input: "save/RestoreV2:332" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_332" op: "AssignVariableOp" input: "bert/encoder/layer_4/attention/self/value/kernel/adam_v" input: "save/Identity_333" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_334" op: "Identity" input: "save/RestoreV2:333" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_333" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias" input: "save/Identity_334" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_335" op: "Identity" input: "save/RestoreV2:334" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_334" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_m" input: "save/Identity_335" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_336" op: "Identity" input: "save/RestoreV2:335" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_335" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/bias/adam_v" input: "save/Identity_336" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_337" op: "Identity" input: "save/RestoreV2:336" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_336" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel" input: "save/Identity_337" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_338" op: "Identity" input: "save/RestoreV2:337" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_337" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_m" input: "save/Identity_338" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_339" op: "Identity" input: "save/RestoreV2:338" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_338" op: "AssignVariableOp" input: "bert/encoder/layer_4/intermediate/dense/kernel/adam_v" input: "save/Identity_339" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_340" op: "Identity" input: "save/RestoreV2:339" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_339" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/bias" input: "save/Identity_340" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_341" op: "Identity" input: "save/RestoreV2:340" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_340" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/bias/adam_m" input: "save/Identity_341" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_342" op: "Identity" input: "save/RestoreV2:341" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_341" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/bias/adam_v" input: "save/Identity_342" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_343" op: "Identity" input: "save/RestoreV2:342" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_342" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/kernel" input: "save/Identity_343" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_344" op: "Identity" input: "save/RestoreV2:343" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_343" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_m" input: "save/Identity_344" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_345" op: "Identity" input: "save/RestoreV2:344" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_344" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/dense/kernel/adam_v" input: "save/Identity_345" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_346" op: "Identity" input: "save/RestoreV2:345" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_345" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta" input: "save/Identity_346" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_347" op: "Identity" input: "save/RestoreV2:346" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_346" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_m" input: "save/Identity_347" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_348" op: "Identity" input: "save/RestoreV2:347" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_347" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/beta/adam_v" input: "save/Identity_348" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_349" op: "Identity" input: "save/RestoreV2:348" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_348" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma" input: "save/Identity_349" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_350" op: "Identity" input: "save/RestoreV2:349" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_349" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_m" input: "save/Identity_350" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_351" op: "Identity" input: "save/RestoreV2:350" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_350" op: "AssignVariableOp" input: "bert/encoder/layer_4/output/layer_normalization_10/gamma/adam_v" input: "save/Identity_351" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_352" op: "Identity" input: "save/RestoreV2:351" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_351" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias" input: "save/Identity_352" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_353" op: "Identity" input: "save/RestoreV2:352" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_352" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_m" input: "save/Identity_353" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_354" op: "Identity" input: "save/RestoreV2:353" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_353" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/bias/adam_v" input: "save/Identity_354" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_355" op: "Identity" input: "save/RestoreV2:354" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_354" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel" input: "save/Identity_355" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_356" op: "Identity" input: "save/RestoreV2:355" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_355" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_m" input: "save/Identity_356" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_357" op: "Identity" input: "save/RestoreV2:356" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_356" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/dense/kernel/adam_v" input: "save/Identity_357" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_358" op: "Identity" input: "save/RestoreV2:357" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_357" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta" input: "save/Identity_358" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_359" op: "Identity" input: "save/RestoreV2:358" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_358" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_m" input: "save/Identity_359" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_360" op: "Identity" input: "save/RestoreV2:359" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_359" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/beta/adam_v" input: "save/Identity_360" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_361" op: "Identity" input: "save/RestoreV2:360" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_360" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma" input: "save/Identity_361" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_362" op: "Identity" input: "save/RestoreV2:361" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_361" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_m" input: "save/Identity_362" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_363" op: "Identity" input: "save/RestoreV2:362" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_362" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/output/layer_normalization_11/gamma/adam_v" input: "save/Identity_363" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_364" op: "Identity" input: "save/RestoreV2:363" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_363" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias" input: "save/Identity_364" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_365" op: "Identity" input: "save/RestoreV2:364" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_364" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_m" input: "save/Identity_365" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_366" op: "Identity" input: "save/RestoreV2:365" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_365" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/bias/adam_v" input: "save/Identity_366" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_367" op: "Identity" input: "save/RestoreV2:366" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_366" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel" input: "save/Identity_367" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_368" op: "Identity" input: "save/RestoreV2:367" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_367" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_m" input: "save/Identity_368" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_369" op: "Identity" input: "save/RestoreV2:368" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_368" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/key/kernel/adam_v" input: "save/Identity_369" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_370" op: "Identity" input: "save/RestoreV2:369" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_369" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias" input: "save/Identity_370" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_371" op: "Identity" input: "save/RestoreV2:370" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_370" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_m" input: "save/Identity_371" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_372" op: "Identity" input: "save/RestoreV2:371" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_371" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/bias/adam_v" input: "save/Identity_372" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_373" op: "Identity" input: "save/RestoreV2:372" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_372" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel" input: "save/Identity_373" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_374" op: "Identity" input: "save/RestoreV2:373" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_373" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_m" input: "save/Identity_374" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_375" op: "Identity" input: "save/RestoreV2:374" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_374" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/query/kernel/adam_v" input: "save/Identity_375" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_376" op: "Identity" input: "save/RestoreV2:375" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_375" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias" input: "save/Identity_376" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_377" op: "Identity" input: "save/RestoreV2:376" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_376" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_m" input: "save/Identity_377" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_378" op: "Identity" input: "save/RestoreV2:377" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_377" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/bias/adam_v" input: "save/Identity_378" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_379" op: "Identity" input: "save/RestoreV2:378" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_378" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel" input: "save/Identity_379" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_380" op: "Identity" input: "save/RestoreV2:379" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_379" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_m" input: "save/Identity_380" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_381" op: "Identity" input: "save/RestoreV2:380" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_380" op: "AssignVariableOp" input: "bert/encoder/layer_5/attention/self/value/kernel/adam_v" input: "save/Identity_381" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_382" op: "Identity" input: "save/RestoreV2:381" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_381" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias" input: "save/Identity_382" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_383" op: "Identity" input: "save/RestoreV2:382" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_382" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_m" input: "save/Identity_383" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_384" op: "Identity" input: "save/RestoreV2:383" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_383" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/bias/adam_v" input: "save/Identity_384" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_385" op: "Identity" input: "save/RestoreV2:384" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_384" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel" input: "save/Identity_385" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_386" op: "Identity" input: "save/RestoreV2:385" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_385" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_m" input: "save/Identity_386" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_387" op: "Identity" input: "save/RestoreV2:386" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_386" op: "AssignVariableOp" input: "bert/encoder/layer_5/intermediate/dense/kernel/adam_v" input: "save/Identity_387" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_388" op: "Identity" input: "save/RestoreV2:387" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_387" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/bias" input: "save/Identity_388" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_389" op: "Identity" input: "save/RestoreV2:388" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_388" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/bias/adam_m" input: "save/Identity_389" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_390" op: "Identity" input: "save/RestoreV2:389" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_389" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/bias/adam_v" input: "save/Identity_390" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_391" op: "Identity" input: "save/RestoreV2:390" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_390" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/kernel" input: "save/Identity_391" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_392" op: "Identity" input: "save/RestoreV2:391" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_391" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_m" input: "save/Identity_392" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_393" op: "Identity" input: "save/RestoreV2:392" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_392" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/dense/kernel/adam_v" input: "save/Identity_393" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_394" op: "Identity" input: "save/RestoreV2:393" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_393" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta" input: "save/Identity_394" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_395" op: "Identity" input: "save/RestoreV2:394" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_394" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_m" input: "save/Identity_395" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_396" op: "Identity" input: "save/RestoreV2:395" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_395" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/beta/adam_v" input: "save/Identity_396" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_397" op: "Identity" input: "save/RestoreV2:396" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_396" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma" input: "save/Identity_397" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_398" op: "Identity" input: "save/RestoreV2:397" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_397" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_m" input: "save/Identity_398" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_399" op: "Identity" input: "save/RestoreV2:398" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_398" op: "AssignVariableOp" input: "bert/encoder/layer_5/output/layer_normalization_12/gamma/adam_v" input: "save/Identity_399" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_400" op: "Identity" input: "save/RestoreV2:399" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_399" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias" input: "save/Identity_400" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_401" op: "Identity" input: "save/RestoreV2:400" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_400" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_m" input: "save/Identity_401" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_402" op: "Identity" input: "save/RestoreV2:401" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_401" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/bias/adam_v" input: "save/Identity_402" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_403" op: "Identity" input: "save/RestoreV2:402" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_402" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel" input: "save/Identity_403" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_404" op: "Identity" input: "save/RestoreV2:403" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_403" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_m" input: "save/Identity_404" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_405" op: "Identity" input: "save/RestoreV2:404" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_404" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/dense/kernel/adam_v" input: "save/Identity_405" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_406" op: "Identity" input: "save/RestoreV2:405" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_405" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta" input: "save/Identity_406" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_407" op: "Identity" input: "save/RestoreV2:406" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_406" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_m" input: "save/Identity_407" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_408" op: "Identity" input: "save/RestoreV2:407" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_407" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/beta/adam_v" input: "save/Identity_408" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_409" op: "Identity" input: "save/RestoreV2:408" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_408" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma" input: "save/Identity_409" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_410" op: "Identity" input: "save/RestoreV2:409" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_409" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_m" input: "save/Identity_410" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_411" op: "Identity" input: "save/RestoreV2:410" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_410" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/output/layer_normalization_13/gamma/adam_v" input: "save/Identity_411" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_412" op: "Identity" input: "save/RestoreV2:411" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_411" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias" input: "save/Identity_412" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_413" op: "Identity" input: "save/RestoreV2:412" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_412" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_m" input: "save/Identity_413" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_414" op: "Identity" input: "save/RestoreV2:413" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_413" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/bias/adam_v" input: "save/Identity_414" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_415" op: "Identity" input: "save/RestoreV2:414" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_414" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel" input: "save/Identity_415" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_416" op: "Identity" input: "save/RestoreV2:415" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_415" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_m" input: "save/Identity_416" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_417" op: "Identity" input: "save/RestoreV2:416" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_416" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/key/kernel/adam_v" input: "save/Identity_417" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_418" op: "Identity" input: "save/RestoreV2:417" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_417" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias" input: "save/Identity_418" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_419" op: "Identity" input: "save/RestoreV2:418" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_418" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_m" input: "save/Identity_419" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_420" op: "Identity" input: "save/RestoreV2:419" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_419" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/bias/adam_v" input: "save/Identity_420" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_421" op: "Identity" input: "save/RestoreV2:420" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_420" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel" input: "save/Identity_421" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_422" op: "Identity" input: "save/RestoreV2:421" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_421" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_m" input: "save/Identity_422" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_423" op: "Identity" input: "save/RestoreV2:422" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_422" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/query/kernel/adam_v" input: "save/Identity_423" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_424" op: "Identity" input: "save/RestoreV2:423" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_423" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias" input: "save/Identity_424" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_425" op: "Identity" input: "save/RestoreV2:424" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_424" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_m" input: "save/Identity_425" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_426" op: "Identity" input: "save/RestoreV2:425" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_425" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/bias/adam_v" input: "save/Identity_426" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_427" op: "Identity" input: "save/RestoreV2:426" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_426" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel" input: "save/Identity_427" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_428" op: "Identity" input: "save/RestoreV2:427" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_427" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_m" input: "save/Identity_428" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_429" op: "Identity" input: "save/RestoreV2:428" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_428" op: "AssignVariableOp" input: "bert/encoder/layer_6/attention/self/value/kernel/adam_v" input: "save/Identity_429" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_430" op: "Identity" input: "save/RestoreV2:429" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_429" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias" input: "save/Identity_430" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_431" op: "Identity" input: "save/RestoreV2:430" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_430" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_m" input: "save/Identity_431" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_432" op: "Identity" input: "save/RestoreV2:431" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_431" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/bias/adam_v" input: "save/Identity_432" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_433" op: "Identity" input: "save/RestoreV2:432" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_432" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel" input: "save/Identity_433" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_434" op: "Identity" input: "save/RestoreV2:433" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_433" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_m" input: "save/Identity_434" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_435" op: "Identity" input: "save/RestoreV2:434" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_434" op: "AssignVariableOp" input: "bert/encoder/layer_6/intermediate/dense/kernel/adam_v" input: "save/Identity_435" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_436" op: "Identity" input: "save/RestoreV2:435" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_435" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/bias" input: "save/Identity_436" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_437" op: "Identity" input: "save/RestoreV2:436" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_436" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/bias/adam_m" input: "save/Identity_437" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_438" op: "Identity" input: "save/RestoreV2:437" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_437" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/bias/adam_v" input: "save/Identity_438" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_439" op: "Identity" input: "save/RestoreV2:438" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_438" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/kernel" input: "save/Identity_439" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_440" op: "Identity" input: "save/RestoreV2:439" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_439" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_m" input: "save/Identity_440" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_441" op: "Identity" input: "save/RestoreV2:440" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_440" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/dense/kernel/adam_v" input: "save/Identity_441" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_442" op: "Identity" input: "save/RestoreV2:441" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_441" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta" input: "save/Identity_442" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_443" op: "Identity" input: "save/RestoreV2:442" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_442" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_m" input: "save/Identity_443" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_444" op: "Identity" input: "save/RestoreV2:443" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_443" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/beta/adam_v" input: "save/Identity_444" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_445" op: "Identity" input: "save/RestoreV2:444" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_444" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma" input: "save/Identity_445" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_446" op: "Identity" input: "save/RestoreV2:445" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_445" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_m" input: "save/Identity_446" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_447" op: "Identity" input: "save/RestoreV2:446" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_446" op: "AssignVariableOp" input: "bert/encoder/layer_6/output/layer_normalization_14/gamma/adam_v" input: "save/Identity_447" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_448" op: "Identity" input: "save/RestoreV2:447" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_447" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias" input: "save/Identity_448" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_449" op: "Identity" input: "save/RestoreV2:448" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_448" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_m" input: "save/Identity_449" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_450" op: "Identity" input: "save/RestoreV2:449" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_449" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/bias/adam_v" input: "save/Identity_450" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_451" op: "Identity" input: "save/RestoreV2:450" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_450" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel" input: "save/Identity_451" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_452" op: "Identity" input: "save/RestoreV2:451" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_451" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_m" input: "save/Identity_452" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_453" op: "Identity" input: "save/RestoreV2:452" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_452" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/dense/kernel/adam_v" input: "save/Identity_453" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_454" op: "Identity" input: "save/RestoreV2:453" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_453" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta" input: "save/Identity_454" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_455" op: "Identity" input: "save/RestoreV2:454" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_454" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_m" input: "save/Identity_455" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_456" op: "Identity" input: "save/RestoreV2:455" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_455" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/beta/adam_v" input: "save/Identity_456" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_457" op: "Identity" input: "save/RestoreV2:456" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_456" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma" input: "save/Identity_457" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_458" op: "Identity" input: "save/RestoreV2:457" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_457" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_m" input: "save/Identity_458" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_459" op: "Identity" input: "save/RestoreV2:458" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_458" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/output/layer_normalization_15/gamma/adam_v" input: "save/Identity_459" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_460" op: "Identity" input: "save/RestoreV2:459" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_459" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias" input: "save/Identity_460" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_461" op: "Identity" input: "save/RestoreV2:460" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_460" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_m" input: "save/Identity_461" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_462" op: "Identity" input: "save/RestoreV2:461" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_461" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/bias/adam_v" input: "save/Identity_462" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_463" op: "Identity" input: "save/RestoreV2:462" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_462" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel" input: "save/Identity_463" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_464" op: "Identity" input: "save/RestoreV2:463" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_463" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_m" input: "save/Identity_464" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_465" op: "Identity" input: "save/RestoreV2:464" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_464" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/key/kernel/adam_v" input: "save/Identity_465" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_466" op: "Identity" input: "save/RestoreV2:465" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_465" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias" input: "save/Identity_466" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_467" op: "Identity" input: "save/RestoreV2:466" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_466" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_m" input: "save/Identity_467" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_468" op: "Identity" input: "save/RestoreV2:467" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_467" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/bias/adam_v" input: "save/Identity_468" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_469" op: "Identity" input: "save/RestoreV2:468" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_468" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel" input: "save/Identity_469" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_470" op: "Identity" input: "save/RestoreV2:469" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_469" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_m" input: "save/Identity_470" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_471" op: "Identity" input: "save/RestoreV2:470" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_470" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/query/kernel/adam_v" input: "save/Identity_471" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_472" op: "Identity" input: "save/RestoreV2:471" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_471" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias" input: "save/Identity_472" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_473" op: "Identity" input: "save/RestoreV2:472" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_472" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_m" input: "save/Identity_473" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_474" op: "Identity" input: "save/RestoreV2:473" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_473" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/bias/adam_v" input: "save/Identity_474" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_475" op: "Identity" input: "save/RestoreV2:474" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_474" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel" input: "save/Identity_475" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_476" op: "Identity" input: "save/RestoreV2:475" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_475" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_m" input: "save/Identity_476" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_477" op: "Identity" input: "save/RestoreV2:476" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_476" op: "AssignVariableOp" input: "bert/encoder/layer_7/attention/self/value/kernel/adam_v" input: "save/Identity_477" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_478" op: "Identity" input: "save/RestoreV2:477" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_477" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias" input: "save/Identity_478" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_479" op: "Identity" input: "save/RestoreV2:478" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_478" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_m" input: "save/Identity_479" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_480" op: "Identity" input: "save/RestoreV2:479" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_479" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/bias/adam_v" input: "save/Identity_480" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_481" op: "Identity" input: "save/RestoreV2:480" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_480" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel" input: "save/Identity_481" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_482" op: "Identity" input: "save/RestoreV2:481" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_481" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_m" input: "save/Identity_482" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_483" op: "Identity" input: "save/RestoreV2:482" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_482" op: "AssignVariableOp" input: "bert/encoder/layer_7/intermediate/dense/kernel/adam_v" input: "save/Identity_483" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_484" op: "Identity" input: "save/RestoreV2:483" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_483" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/bias" input: "save/Identity_484" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_485" op: "Identity" input: "save/RestoreV2:484" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_484" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/bias/adam_m" input: "save/Identity_485" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_486" op: "Identity" input: "save/RestoreV2:485" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_485" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/bias/adam_v" input: "save/Identity_486" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_487" op: "Identity" input: "save/RestoreV2:486" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_486" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/kernel" input: "save/Identity_487" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_488" op: "Identity" input: "save/RestoreV2:487" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_487" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_m" input: "save/Identity_488" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_489" op: "Identity" input: "save/RestoreV2:488" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_488" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/dense/kernel/adam_v" input: "save/Identity_489" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_490" op: "Identity" input: "save/RestoreV2:489" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_489" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta" input: "save/Identity_490" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_491" op: "Identity" input: "save/RestoreV2:490" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_490" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_m" input: "save/Identity_491" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_492" op: "Identity" input: "save/RestoreV2:491" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_491" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/beta/adam_v" input: "save/Identity_492" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_493" op: "Identity" input: "save/RestoreV2:492" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_492" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma" input: "save/Identity_493" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_494" op: "Identity" input: "save/RestoreV2:493" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_493" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_m" input: "save/Identity_494" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_495" op: "Identity" input: "save/RestoreV2:494" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_494" op: "AssignVariableOp" input: "bert/encoder/layer_7/output/layer_normalization_16/gamma/adam_v" input: "save/Identity_495" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_496" op: "Identity" input: "save/RestoreV2:495" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_495" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias" input: "save/Identity_496" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_497" op: "Identity" input: "save/RestoreV2:496" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_496" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_m" input: "save/Identity_497" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_498" op: "Identity" input: "save/RestoreV2:497" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_497" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/bias/adam_v" input: "save/Identity_498" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_499" op: "Identity" input: "save/RestoreV2:498" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_498" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel" input: "save/Identity_499" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_500" op: "Identity" input: "save/RestoreV2:499" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_499" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_m" input: "save/Identity_500" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_501" op: "Identity" input: "save/RestoreV2:500" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_500" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/dense/kernel/adam_v" input: "save/Identity_501" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_502" op: "Identity" input: "save/RestoreV2:501" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_501" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta" input: "save/Identity_502" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_503" op: "Identity" input: "save/RestoreV2:502" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_502" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_m" input: "save/Identity_503" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_504" op: "Identity" input: "save/RestoreV2:503" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_503" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/beta/adam_v" input: "save/Identity_504" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_505" op: "Identity" input: "save/RestoreV2:504" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_504" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma" input: "save/Identity_505" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_506" op: "Identity" input: "save/RestoreV2:505" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_505" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_m" input: "save/Identity_506" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_507" op: "Identity" input: "save/RestoreV2:506" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_506" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/output/layer_normalization_17/gamma/adam_v" input: "save/Identity_507" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_508" op: "Identity" input: "save/RestoreV2:507" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_507" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias" input: "save/Identity_508" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_509" op: "Identity" input: "save/RestoreV2:508" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_508" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_m" input: "save/Identity_509" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_510" op: "Identity" input: "save/RestoreV2:509" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_509" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/bias/adam_v" input: "save/Identity_510" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_511" op: "Identity" input: "save/RestoreV2:510" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_510" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel" input: "save/Identity_511" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_512" op: "Identity" input: "save/RestoreV2:511" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_511" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_m" input: "save/Identity_512" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_513" op: "Identity" input: "save/RestoreV2:512" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_512" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/key/kernel/adam_v" input: "save/Identity_513" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_514" op: "Identity" input: "save/RestoreV2:513" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_513" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias" input: "save/Identity_514" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_515" op: "Identity" input: "save/RestoreV2:514" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_514" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_m" input: "save/Identity_515" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_516" op: "Identity" input: "save/RestoreV2:515" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_515" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/bias/adam_v" input: "save/Identity_516" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_517" op: "Identity" input: "save/RestoreV2:516" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_516" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel" input: "save/Identity_517" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_518" op: "Identity" input: "save/RestoreV2:517" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_517" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_m" input: "save/Identity_518" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_519" op: "Identity" input: "save/RestoreV2:518" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_518" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/query/kernel/adam_v" input: "save/Identity_519" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_520" op: "Identity" input: "save/RestoreV2:519" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_519" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias" input: "save/Identity_520" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_521" op: "Identity" input: "save/RestoreV2:520" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_520" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_m" input: "save/Identity_521" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_522" op: "Identity" input: "save/RestoreV2:521" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_521" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/bias/adam_v" input: "save/Identity_522" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_523" op: "Identity" input: "save/RestoreV2:522" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_522" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel" input: "save/Identity_523" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_524" op: "Identity" input: "save/RestoreV2:523" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_523" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_m" input: "save/Identity_524" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_525" op: "Identity" input: "save/RestoreV2:524" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_524" op: "AssignVariableOp" input: "bert/encoder/layer_8/attention/self/value/kernel/adam_v" input: "save/Identity_525" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_526" op: "Identity" input: "save/RestoreV2:525" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_525" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias" input: "save/Identity_526" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_527" op: "Identity" input: "save/RestoreV2:526" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_526" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_m" input: "save/Identity_527" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_528" op: "Identity" input: "save/RestoreV2:527" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_527" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/bias/adam_v" input: "save/Identity_528" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_529" op: "Identity" input: "save/RestoreV2:528" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_528" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel" input: "save/Identity_529" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_530" op: "Identity" input: "save/RestoreV2:529" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_529" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_m" input: "save/Identity_530" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_531" op: "Identity" input: "save/RestoreV2:530" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_530" op: "AssignVariableOp" input: "bert/encoder/layer_8/intermediate/dense/kernel/adam_v" input: "save/Identity_531" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_532" op: "Identity" input: "save/RestoreV2:531" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_531" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/bias" input: "save/Identity_532" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_533" op: "Identity" input: "save/RestoreV2:532" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_532" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/bias/adam_m" input: "save/Identity_533" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_534" op: "Identity" input: "save/RestoreV2:533" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_533" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/bias/adam_v" input: "save/Identity_534" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_535" op: "Identity" input: "save/RestoreV2:534" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_534" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/kernel" input: "save/Identity_535" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_536" op: "Identity" input: "save/RestoreV2:535" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_535" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_m" input: "save/Identity_536" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_537" op: "Identity" input: "save/RestoreV2:536" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_536" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/dense/kernel/adam_v" input: "save/Identity_537" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_538" op: "Identity" input: "save/RestoreV2:537" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_537" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta" input: "save/Identity_538" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_539" op: "Identity" input: "save/RestoreV2:538" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_538" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_m" input: "save/Identity_539" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_540" op: "Identity" input: "save/RestoreV2:539" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_539" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/beta/adam_v" input: "save/Identity_540" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_541" op: "Identity" input: "save/RestoreV2:540" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_540" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma" input: "save/Identity_541" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_542" op: "Identity" input: "save/RestoreV2:541" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_541" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_m" input: "save/Identity_542" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_543" op: "Identity" input: "save/RestoreV2:542" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_542" op: "AssignVariableOp" input: "bert/encoder/layer_8/output/layer_normalization_18/gamma/adam_v" input: "save/Identity_543" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_544" op: "Identity" input: "save/RestoreV2:543" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_543" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias" input: "save/Identity_544" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_545" op: "Identity" input: "save/RestoreV2:544" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_544" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_m" input: "save/Identity_545" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_546" op: "Identity" input: "save/RestoreV2:545" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_545" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/bias/adam_v" input: "save/Identity_546" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_547" op: "Identity" input: "save/RestoreV2:546" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_546" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel" input: "save/Identity_547" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_548" op: "Identity" input: "save/RestoreV2:547" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_547" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_m" input: "save/Identity_548" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_549" op: "Identity" input: "save/RestoreV2:548" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_548" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/dense/kernel/adam_v" input: "save/Identity_549" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_550" op: "Identity" input: "save/RestoreV2:549" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_549" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta" input: "save/Identity_550" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_551" op: "Identity" input: "save/RestoreV2:550" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_550" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_m" input: "save/Identity_551" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_552" op: "Identity" input: "save/RestoreV2:551" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_551" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/beta/adam_v" input: "save/Identity_552" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_553" op: "Identity" input: "save/RestoreV2:552" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_552" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma" input: "save/Identity_553" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_554" op: "Identity" input: "save/RestoreV2:553" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_553" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_m" input: "save/Identity_554" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_555" op: "Identity" input: "save/RestoreV2:554" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_554" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/output/layer_normalization_19/gamma/adam_v" input: "save/Identity_555" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_556" op: "Identity" input: "save/RestoreV2:555" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_555" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias" input: "save/Identity_556" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_557" op: "Identity" input: "save/RestoreV2:556" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_556" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_m" input: "save/Identity_557" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_558" op: "Identity" input: "save/RestoreV2:557" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_557" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/bias/adam_v" input: "save/Identity_558" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_559" op: "Identity" input: "save/RestoreV2:558" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_558" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel" input: "save/Identity_559" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_560" op: "Identity" input: "save/RestoreV2:559" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_559" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_m" input: "save/Identity_560" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_561" op: "Identity" input: "save/RestoreV2:560" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_560" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/key/kernel/adam_v" input: "save/Identity_561" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_562" op: "Identity" input: "save/RestoreV2:561" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_561" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias" input: "save/Identity_562" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_563" op: "Identity" input: "save/RestoreV2:562" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_562" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_m" input: "save/Identity_563" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_564" op: "Identity" input: "save/RestoreV2:563" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_563" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/bias/adam_v" input: "save/Identity_564" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_565" op: "Identity" input: "save/RestoreV2:564" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_564" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel" input: "save/Identity_565" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_566" op: "Identity" input: "save/RestoreV2:565" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_565" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_m" input: "save/Identity_566" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_567" op: "Identity" input: "save/RestoreV2:566" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_566" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/query/kernel/adam_v" input: "save/Identity_567" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_568" op: "Identity" input: "save/RestoreV2:567" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_567" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias" input: "save/Identity_568" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_569" op: "Identity" input: "save/RestoreV2:568" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_568" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_m" input: "save/Identity_569" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_570" op: "Identity" input: "save/RestoreV2:569" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_569" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/bias/adam_v" input: "save/Identity_570" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_571" op: "Identity" input: "save/RestoreV2:570" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_570" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel" input: "save/Identity_571" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_572" op: "Identity" input: "save/RestoreV2:571" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_571" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_m" input: "save/Identity_572" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_573" op: "Identity" input: "save/RestoreV2:572" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_572" op: "AssignVariableOp" input: "bert/encoder/layer_9/attention/self/value/kernel/adam_v" input: "save/Identity_573" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_574" op: "Identity" input: "save/RestoreV2:573" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_573" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias" input: "save/Identity_574" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_575" op: "Identity" input: "save/RestoreV2:574" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_574" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_m" input: "save/Identity_575" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_576" op: "Identity" input: "save/RestoreV2:575" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_575" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/bias/adam_v" input: "save/Identity_576" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_577" op: "Identity" input: "save/RestoreV2:576" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_576" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel" input: "save/Identity_577" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_578" op: "Identity" input: "save/RestoreV2:577" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_577" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_m" input: "save/Identity_578" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_579" op: "Identity" input: "save/RestoreV2:578" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_578" op: "AssignVariableOp" input: "bert/encoder/layer_9/intermediate/dense/kernel/adam_v" input: "save/Identity_579" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_580" op: "Identity" input: "save/RestoreV2:579" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_579" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/bias" input: "save/Identity_580" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_581" op: "Identity" input: "save/RestoreV2:580" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_580" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/bias/adam_m" input: "save/Identity_581" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_582" op: "Identity" input: "save/RestoreV2:581" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_581" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/bias/adam_v" input: "save/Identity_582" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_583" op: "Identity" input: "save/RestoreV2:582" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_582" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/kernel" input: "save/Identity_583" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_584" op: "Identity" input: "save/RestoreV2:583" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_583" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_m" input: "save/Identity_584" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_585" op: "Identity" input: "save/RestoreV2:584" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_584" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/dense/kernel/adam_v" input: "save/Identity_585" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_586" op: "Identity" input: "save/RestoreV2:585" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_585" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta" input: "save/Identity_586" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_587" op: "Identity" input: "save/RestoreV2:586" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_586" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_m" input: "save/Identity_587" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_588" op: "Identity" input: "save/RestoreV2:587" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_587" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/beta/adam_v" input: "save/Identity_588" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_589" op: "Identity" input: "save/RestoreV2:588" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_588" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma" input: "save/Identity_589" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_590" op: "Identity" input: "save/RestoreV2:589" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_589" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_m" input: "save/Identity_590" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_591" op: "Identity" input: "save/RestoreV2:590" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_590" op: "AssignVariableOp" input: "bert/encoder/layer_9/output/layer_normalization_20/gamma/adam_v" input: "save/Identity_591" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_592" op: "Identity" input: "save/RestoreV2:591" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_591" op: "AssignVariableOp" input: "bert/pooler/dense/bias" input: "save/Identity_592" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_593" op: "Identity" input: "save/RestoreV2:592" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_592" op: "AssignVariableOp" input: "bert/pooler/dense/bias/adam_m" input: "save/Identity_593" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_594" op: "Identity" input: "save/RestoreV2:593" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_593" op: "AssignVariableOp" input: "bert/pooler/dense/bias/adam_v" input: "save/Identity_594" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_595" op: "Identity" input: "save/RestoreV2:594" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_594" op: "AssignVariableOp" input: "bert/pooler/dense/kernel" input: "save/Identity_595" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_596" op: "Identity" input: "save/RestoreV2:595" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_595" op: "AssignVariableOp" input: "bert/pooler/dense/kernel/adam_m" input: "save/Identity_596" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_597" op: "Identity" input: "save/RestoreV2:596" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_596" op: "AssignVariableOp" input: "bert/pooler/dense/kernel/adam_v" input: "save/Identity_597" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_598" op: "Identity" input: "save/RestoreV2:597" attr { key: "T" value { type: DT_INT64 } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_597" op: "AssignVariableOp" input: "global_step" input: "save/Identity_598" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_599" op: "Identity" input: "save/RestoreV2:598" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_598" op: "AssignVariableOp" input: "output_bias" input: "save/Identity_599" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_600" op: "Identity" input: "save/RestoreV2:599" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_599" op: "AssignVariableOp" input: "output_bias/adam_m" input: "save/Identity_600" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_601" op: "Identity" input: "save/RestoreV2:600" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_600" op: "AssignVariableOp" input: "output_bias/adam_v" input: "save/Identity_601" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_602" op: "Identity" input: "save/RestoreV2:601" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_601" op: "AssignVariableOp" input: "output_weights" input: "save/Identity_602" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_603" op: "Identity" input: "save/RestoreV2:602" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_602" op: "AssignVariableOp" input: "output_weights/adam_m" input: "save/Identity_603" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/Identity_604" op: "Identity" input: "save/RestoreV2:603" attr { key: "T" value { type: DT_FLOAT } } attr { key: "_output_shapes" value { list { shape { unknown_rank: true } } } } } node { name: "save/AssignVariableOp_603" op: "AssignVariableOp" input: "output_weights/adam_v" input: "save/Identity_604" attr { key: "_has_manual_control_dependencies" value { b: true } } attr { key: "dtype" value { type: DT_FLOAT } } attr { key: "validate_shape" value { b: false } } } node { name: "save/restore_shard" op: "NoOp" input: "^save/AssignVariableOp" input: "^save/AssignVariableOp_1" input: "^save/AssignVariableOp_10" input: "^save/AssignVariableOp_100" input: "^save/AssignVariableOp_101" input: "^save/AssignVariableOp_102" input: "^save/AssignVariableOp_103" input: "^save/AssignVariableOp_104" input: "^save/AssignVariableOp_105" input: "^save/AssignVariableOp_106" input: "^save/AssignVariableOp_107" input: "^save/AssignVariableOp_108" input: "^save/AssignVariableOp_109" input: "^save/AssignVariableOp_11" input: "^save/AssignVariableOp_110" input: "^save/AssignVariableOp_111" input: "^save/AssignVariableOp_112" input: "^save/AssignVariableOp_113" input: "^save/AssignVariableOp_114" input: "^save/AssignVariableOp_115" input: "^save/AssignVariableOp_116" input: "^save/AssignVariableOp_117" input: "^save/AssignVariableOp_118" input: "^save/AssignVariableOp_119" input: "^save/AssignVariableOp_12" input: "^save/AssignVariableOp_120" input: "^save/AssignVariableOp_121" input: "^save/AssignVariableOp_122" input: "^save/AssignVariableOp_123" input: "^save/AssignVariableOp_124" input: "^save/AssignVariableOp_125" input: "^save/AssignVariableOp_126" input: "^save/AssignVariableOp_127" input: "^save/AssignVariableOp_128" input: "^save/AssignVariableOp_129" input: "^save/AssignVariableOp_13" input: "^save/AssignVariableOp_130" input: "^save/AssignVariableOp_131" input: "^save/AssignVariableOp_132" input: "^save/AssignVariableOp_133" input: "^save/AssignVariableOp_134" input: "^save/AssignVariableOp_135" input: "^save/AssignVariableOp_136" input: "^save/AssignVariableOp_137" input: "^save/AssignVariableOp_138" input: "^save/AssignVariableOp_139" input: "^save/AssignVariableOp_14" input: "^save/AssignVariableOp_140" input: "^save/AssignVariableOp_141" input: "^save/AssignVariableOp_142" input: "^save/AssignVariableOp_143" input: "^save/AssignVariableOp_144" input: "^save/AssignVariableOp_145" input: "^save/AssignVariableOp_146" input: "^save/AssignVariableOp_147" input: "^save/AssignVariableOp_148" input: "^save/AssignVariableOp_149" input: "^save/AssignVariableOp_15" input: "^save/AssignVariableOp_150" input: "^save/AssignVariableOp_151" input: "^save/AssignVariableOp_152" input: "^save/AssignVariableOp_153" input: "^save/AssignVariableOp_154" input: "^save/AssignVariableOp_155" input: "^save/AssignVariableOp_156" input: "^save/AssignVariableOp_157" input: "^save/AssignVariableOp_158" input: "^save/AssignVariableOp_159" input: "^save/AssignVariableOp_16" input: "^save/AssignVariableOp_160" input: "^save/AssignVariableOp_161" input: "^save/AssignVariableOp_162" input: "^save/AssignVariableOp_163" input: "^save/AssignVariableOp_164" input: "^save/AssignVariableOp_165" input: "^save/AssignVariableOp_166" input: "^save/AssignVariableOp_167" input: "^save/AssignVariableOp_168" input: "^save/AssignVariableOp_169" input: "^save/AssignVariableOp_17" input: "^save/AssignVariableOp_170" input: "^save/AssignVariableOp_171" input: "^save/AssignVariableOp_172" input: "^save/AssignVariableOp_173" input: "^save/AssignVariableOp_174" input: "^save/AssignVariableOp_175" input: "^save/AssignVariableOp_176" input: "^save/AssignVariableOp_177" input: "^save/AssignVariableOp_178" input: "^save/AssignVariableOp_179" input: "^save/AssignVariableOp_18" input: "^save/AssignVariableOp_180" input: "^save/AssignVariableOp_181" input: "^save/AssignVariableOp_182" input: "^save/AssignVariableOp_183" input: "^save/AssignVariableOp_184" input: "^save/AssignVariableOp_185" input: "^save/AssignVariableOp_186" input: "^save/AssignVariableOp_187" input: "^save/AssignVariableOp_188" input: "^save/AssignVariableOp_189" input: "^save/AssignVariableOp_19" input: "^save/AssignVariableOp_190" input: "^save/AssignVariableOp_191" input: "^save/AssignVariableOp_192" input: "^save/AssignVariableOp_193" input: "^save/AssignVariableOp_194" input: "^save/AssignVariableOp_195" input: "^save/AssignVariableOp_196" input: "^save/AssignVariableOp_197" input: "^save/AssignVariableOp_198" input: "^save/AssignVariableOp_199" input: "^save/AssignVariableOp_2" input: "^save/AssignVariableOp_20" input: "^save/AssignVariableOp_200" input: "^save/AssignVariableOp_201" input: "^save/AssignVariableOp_202" input: "^save/AssignVariableOp_203" input: "^save/AssignVariableOp_204" input: "^save/AssignVariableOp_205" input: "^save/AssignVariableOp_206" input: "^save/AssignVariableOp_207" input: "^save/AssignVariableOp_208" input: "^save/AssignVariableOp_209" input: "^save/AssignVariableOp_21" input: "^save/AssignVariableOp_210" input: "^save/AssignVariableOp_211" input: "^save/AssignVariableOp_212" input: "^save/AssignVariableOp_213" input: "^save/AssignVariableOp_214" input: "^save/AssignVariableOp_215" input: "^save/AssignVariableOp_216" input: "^save/AssignVariableOp_217" input: "^save/AssignVariableOp_218" input: "^save/AssignVariableOp_219" input: "^save/AssignVariableOp_22" input: "^save/AssignVariableOp_220" input: "^save/AssignVariableOp_221" input: "^save/AssignVariableOp_222" input: "^save/AssignVariableOp_223" input: "^save/AssignVariableOp_224" input: "^save/AssignVariableOp_225" input: "^save/AssignVariableOp_226" input: "^save/AssignVariableOp_227" input: "^save/AssignVariableOp_228" input: "^save/AssignVariableOp_229" input: "^save/AssignVariableOp_23" input: "^save/AssignVariableOp_230" input: "^save/AssignVariableOp_231" input: "^save/AssignVariableOp_232" input: "^save/AssignVariableOp_233" input: "^save/AssignVariableOp_234" input: "^save/AssignVariableOp_235" input: "^save/AssignVariableOp_236" input: "^save/AssignVariableOp_237" input: "^save/AssignVariableOp_238" input: "^save/AssignVariableOp_239" input: "^save/AssignVariableOp_24" input: "^save/AssignVariableOp_240" input: "^save/AssignVariableOp_241" input: "^save/AssignVariableOp_242" input: "^save/AssignVariableOp_243" input: "^save/AssignVariableOp_244" input: "^save/AssignVariableOp_245" input: "^save/AssignVariableOp_246" input: "^save/AssignVariableOp_247" input: "^save/AssignVariableOp_248" input: "^save/AssignVariableOp_249" input: "^save/AssignVariableOp_25" input: "^save/AssignVariableOp_250" input: "^save/AssignVariableOp_251" input: "^save/AssignVariableOp_252" input: "^save/AssignVariableOp_253" input: "^save/AssignVariableOp_254" input: "^save/AssignVariableOp_255" input: "^save/AssignVariableOp_256" input: "^save/AssignVariableOp_257" input: "^save/AssignVariableOp_258" input: "^save/AssignVariableOp_259" input: "^save/AssignVariableOp_26" input: "^save/AssignVariableOp_260" input: "^save/AssignVariableOp_261" input: "^save/AssignVariableOp_262" input: "^save/AssignVariableOp_263" input: "^save/AssignVariableOp_264" input: "^save/AssignVariableOp_265" input: "^save/AssignVariableOp_266" input: "^save/AssignVariableOp_267" input: "^save/AssignVariableOp_268" input: "^save/AssignVariableOp_269" input: "^save/AssignVariableOp_27" input: "^save/AssignVariableOp_270" input: "^save/AssignVariableOp_271" input: "^save/AssignVariableOp_272" input: "^save/AssignVariableOp_273" input: "^save/AssignVariableOp_274" input: "^save/AssignVariableOp_275" input: "^save/AssignVariableOp_276" input: "^save/AssignVariableOp_277" input: "^save/AssignVariableOp_278" input: "^save/AssignVariableOp_279" input: "^save/AssignVariableOp_28" input: "^save/AssignVariableOp_280" input: "^save/AssignVariableOp_281" input: "^save/AssignVariableOp_282" input: "^save/AssignVariableOp_283" input: "^save/AssignVariableOp_284" input: "^save/AssignVariableOp_285" input: "^save/AssignVariableOp_286" input: "^save/AssignVariableOp_287" input: "^save/AssignVariableOp_288" input: "^save/AssignVariableOp_289" input: "^save/AssignVariableOp_29" input: "^save/AssignVariableOp_290" input: "^save/AssignVariableOp_291" input: "^save/AssignVariableOp_292" input: "^save/AssignVariableOp_293" input: "^save/AssignVariableOp_294" input: "^save/AssignVariableOp_295" input: "^save/AssignVariableOp_296" input: "^save/AssignVariableOp_297" input: "^save/AssignVariableOp_298" input: "^save/AssignVariableOp_299" input: "^save/AssignVariableOp_3" input: "^save/AssignVariableOp_30" input: "^save/AssignVariableOp_300" input: "^save/AssignVariableOp_301" input: "^save/AssignVariableOp_302" input: "^save/AssignVariableOp_303" input: "^save/AssignVariableOp_304" input: "^save/AssignVariableOp_305" input: "^save/AssignVariableOp_306" input: "^save/AssignVariableOp_307" input: "^save/AssignVariableOp_308" input: "^save/AssignVariableOp_309" input: "^save/AssignVariableOp_31" input: "^save/AssignVariableOp_310" input: "^save/AssignVariableOp_311" input: "^save/AssignVariableOp_312" input: "^save/AssignVariableOp_313" input: "^save/AssignVariableOp_314" input: "^save/AssignVariableOp_315" input: "^save/AssignVariableOp_316" input: "^save/AssignVariableOp_317" input: "^save/AssignVariableOp_318" input: "^save/AssignVariableOp_319" input: "^save/AssignVariableOp_32" input: "^save/AssignVariableOp_320" input: "^save/AssignVariableOp_321" input: "^save/AssignVariableOp_322" input: "^save/AssignVariableOp_323" input: "^save/AssignVariableOp_324" input: "^save/AssignVariableOp_325" input: "^save/AssignVariableOp_326" input: "^save/AssignVariableOp_327" input: "^save/AssignVariableOp_328" input: "^save/AssignVariableOp_329" input: "^save/AssignVariableOp_33" input: "^save/AssignVariableOp_330" input: "^save/AssignVariableOp_331" input: "^save/AssignVariableOp_332" input: "^save/AssignVariableOp_333" input: "^save/AssignVariableOp_334" input: "^save/AssignVariableOp_335" input: "^save/AssignVariableOp_336" input: "^save/AssignVariableOp_337" input: "^save/AssignVariableOp_338" input: "^save/AssignVariableOp_339" input: "^save/AssignVariableOp_34" input: "^save/AssignVariableOp_340" input: "^save/AssignVariableOp_341" input: "^save/AssignVariableOp_342" input: "^save/AssignVariableOp_343" input: "^save/AssignVariableOp_344" input: "^save/AssignVariableOp_345" input: "^save/AssignVariableOp_346" input: "^save/AssignVariableOp_347" input: "^save/AssignVariableOp_348" input: "^save/AssignVariableOp_349" input: "^save/AssignVariableOp_35" input: "^save/AssignVariableOp_350" input: "^save/AssignVariableOp_351" input: "^save/AssignVariableOp_352" input: "^save/AssignVariableOp_353" input: "^save/AssignVariableOp_354" input: "^save/AssignVariableOp_355" input: "^save/AssignVariableOp_356" input: "^save/AssignVariableOp_357" input: "^save/AssignVariableOp_358" input: "^save/AssignVariableOp_359" input: "^save/AssignVariableOp_36" input: "^save/AssignVariableOp_360" input: "^save/AssignVariableOp_361" input: "^save/AssignVariableOp_362" input: "^save/AssignVariableOp_363" input: "^save/AssignVariableOp_364" input: "^save/AssignVariableOp_365" input: "^save/AssignVariableOp_366" input: "^save/AssignVariableOp_367" input: "^save/AssignVariableOp_368" input: "^save/AssignVariableOp_369" input: "^save/AssignVariableOp_37" input: "^save/AssignVariableOp_370" input: "^save/AssignVariableOp_371" input: "^save/AssignVariableOp_372" input: "^save/AssignVariableOp_373" input: "^save/AssignVariableOp_374" input: "^save/AssignVariableOp_375" input: "^save/AssignVariableOp_376" input: "^save/AssignVariableOp_377" input: "^save/AssignVariableOp_378" input: "^save/AssignVariableOp_379" input: "^save/AssignVariableOp_38" input: "^save/AssignVariableOp_380" input: "^save/AssignVariableOp_381" input: "^save/AssignVariableOp_382" input: "^save/AssignVariableOp_383" input: "^save/AssignVariableOp_384" input: "^save/AssignVariableOp_385" input: "^save/AssignVariableOp_386" input: "^save/AssignVariableOp_387" input: "^save/AssignVariableOp_388" input: "^save/AssignVariableOp_389" input: "^save/AssignVariableOp_39" input: "^save/AssignVariableOp_390" input: "^save/AssignVariableOp_391" input: "^save/AssignVariableOp_392" input: "^save/AssignVariableOp_393" input: "^save/AssignVariableOp_394" input: "^save/AssignVariableOp_395" input: "^save/AssignVariableOp_396" input: "^save/AssignVariableOp_397" input: "^save/AssignVariableOp_398" input: "^save/AssignVariableOp_399" input: "^save/AssignVariableOp_4" input: "^save/AssignVariableOp_40" input: "^save/AssignVariableOp_400" input: "^save/AssignVariableOp_401" input: "^save/AssignVariableOp_402" input: "^save/AssignVariableOp_403" input: "^save/AssignVariableOp_404" input: "^save/AssignVariableOp_405" input: "^save/AssignVariableOp_406" input: "^save/AssignVariableOp_407" input: "^save/AssignVariableOp_408" input: "^save/AssignVariableOp_409" input: "^save/AssignVariableOp_41" input: "^save/AssignVariableOp_410" input: "^save/AssignVariableOp_411" input: "^save/AssignVariableOp_412" input: "^save/AssignVariableOp_413" input: "^save/AssignVariableOp_414" input: "^save/AssignVariableOp_415" input: "^save/AssignVariableOp_416" input: "^save/AssignVariableOp_417" input: "^save/AssignVariableOp_418" input: "^save/AssignVariableOp_419" input: "^save/AssignVariableOp_42" input: "^save/AssignVariableOp_420" input: "^save/AssignVariableOp_421" input: "^save/AssignVariableOp_422" input: "^save/AssignVariableOp_423" input: "^save/AssignVariableOp_424" input: "^save/AssignVariableOp_425" input: "^save/AssignVariableOp_426" input: "^save/AssignVariableOp_427" input: "^save/AssignVariableOp_428" input: "^save/AssignVariableOp_429" input: "^save/AssignVariableOp_43" input: "^save/AssignVariableOp_430" input: "^save/AssignVariableOp_431" input: "^save/AssignVariableOp_432" input: "^save/AssignVariableOp_433" input: "^save/AssignVariableOp_434" input: "^save/AssignVariableOp_435" input: "^save/AssignVariableOp_436" input: "^save/AssignVariableOp_437" input: "^save/AssignVariableOp_438" input: "^save/AssignVariableOp_439" input: "^save/AssignVariableOp_44" input: "^save/AssignVariableOp_440" input: "^save/AssignVariableOp_441" input: "^save/AssignVariableOp_442" input: "^save/AssignVariableOp_443" input: "^save/AssignVariableOp_444" input: "^save/AssignVariableOp_445" input: "^save/AssignVariableOp_446" input: "^save/AssignVariableOp_447" input: "^save/AssignVariableOp_448" input: "^save/AssignVariableOp_449" input: "^save/AssignVariableOp_45" input: "^save/AssignVariableOp_450" input: "^save/AssignVariableOp_451" input: "^save/AssignVariableOp_452" input: "^save/AssignVariableOp_453" input: "^save/AssignVariableOp_454" input: "^save/AssignVariableOp_455" input: "^save/AssignVariableOp_456" input: "^save/AssignVariableOp_457" input: "^save/AssignVariableOp_458" input: "^save/AssignVariableOp_459" input: "^save/AssignVariableOp_46" input: "^save/AssignVariableOp_460" input: "^save/AssignVariableOp_461" input: "^save/AssignVariableOp_462" input: "^save/AssignVariableOp_463" input: "^save/AssignVariableOp_464" input: "^save/AssignVariableOp_465" input: "^save/AssignVariableOp_466" input: "^save/AssignVariableOp_467" input: "^save/AssignVariableOp_468" input: "^save/AssignVariableOp_469" input: "^save/AssignVariableOp_47" input: "^save/AssignVariableOp_470" input: "^save/AssignVariableOp_471" input: "^save/AssignVariableOp_472" input: "^save/AssignVariableOp_473" input: "^save/AssignVariableOp_474" input: "^save/AssignVariableOp_475" input: "^save/AssignVariableOp_476" input: "^save/AssignVariableOp_477" input: "^save/AssignVariableOp_478" input: "^save/AssignVariableOp_479" input: "^save/AssignVariableOp_48" input: "^save/AssignVariableOp_480" input: "^save/AssignVariableOp_481" input: "^save/AssignVariableOp_482" input: "^save/AssignVariableOp_483" input: "^save/AssignVariableOp_484" input: "^save/AssignVariableOp_485" input: "^save/AssignVariableOp_486" input: "^save/AssignVariableOp_487" input: "^save/AssignVariableOp_488" input: "^save/AssignVariableOp_489" input: "^save/AssignVariableOp_49" input: "^save/AssignVariableOp_490" input: "^save/AssignVariableOp_491" input: "^save/AssignVariableOp_492" input: "^save/AssignVariableOp_493" input: "^save/AssignVariableOp_494" input: "^save/AssignVariableOp_495" input: "^save/AssignVariableOp_496" input: "^save/AssignVariableOp_497" input: "^save/AssignVariableOp_498" input: "^save/AssignVariableOp_499" input: "^save/AssignVariableOp_5" input: "^save/AssignVariableOp_50" input: "^save/AssignVariableOp_500" input: "^save/AssignVariableOp_501" input: "^save/AssignVariableOp_502" input: "^save/AssignVariableOp_503" input: "^save/AssignVariableOp_504" input: "^save/AssignVariableOp_505" input: "^save/AssignVariableOp_506" input: "^save/AssignVariableOp_507" input: "^save/AssignVariableOp_508" input: "^save/AssignVariableOp_509" input: "^save/AssignVariableOp_51" input: "^save/AssignVariableOp_510" input: "^save/AssignVariableOp_511" input: "^save/AssignVariableOp_512" input: "^save/AssignVariableOp_513" input: "^save/AssignVariableOp_514" input: "^save/AssignVariableOp_515" input: "^save/AssignVariableOp_516" input: "^save/AssignVariableOp_517" input: "^save/AssignVariableOp_518" input: "^save/AssignVariableOp_519" input: "^save/AssignVariableOp_52" input: "^save/AssignVariableOp_520" input: "^save/AssignVariableOp_521" input: "^save/AssignVariableOp_522" input: "^save/AssignVariableOp_523" input: "^save/AssignVariableOp_524" input: "^save/AssignVariableOp_525" input: "^save/AssignVariableOp_526" input: "^save/AssignVariableOp_527" input: "^save/AssignVariableOp_528" input: "^save/AssignVariableOp_529" input: "^save/AssignVariableOp_53" input: "^save/AssignVariableOp_530" input: "^save/AssignVariableOp_531" input: "^save/AssignVariableOp_532" input: "^save/AssignVariableOp_533" input: "^save/AssignVariableOp_534" input: "^save/AssignVariableOp_535" input: "^save/AssignVariableOp_536" input: "^save/AssignVariableOp_537" input: "^save/AssignVariableOp_538" input: "^save/AssignVariableOp_539" input: "^save/AssignVariableOp_54" input: "^save/AssignVariableOp_540" input: "^save/AssignVariableOp_541" input: "^save/AssignVariableOp_542" input: "^save/AssignVariableOp_543" input: "^save/AssignVariableOp_544" input: "^save/AssignVariableOp_545" input: "^save/AssignVariableOp_546" input: "^save/AssignVariableOp_547" input: "^save/AssignVariableOp_548" input: "^save/AssignVariableOp_549" input: "^save/AssignVariableOp_55" input: "^save/AssignVariableOp_550" input: "^save/AssignVariableOp_551" input: "^save/AssignVariableOp_552" input: "^save/AssignVariableOp_553" input: "^save/AssignVariableOp_554" input: "^save/AssignVariableOp_555" input: "^save/AssignVariableOp_556" input: "^save/AssignVariableOp_557" input: "^save/AssignVariableOp_558" input: "^save/AssignVariableOp_559" input: "^save/AssignVariableOp_56" input: "^save/AssignVariableOp_560" input: "^save/AssignVariableOp_561" input: "^save/AssignVariableOp_562" input: "^save/AssignVariableOp_563" input: "^save/AssignVariableOp_564" input: "^save/AssignVariableOp_565" input: "^save/AssignVariableOp_566" input: "^save/AssignVariableOp_567" input: "^save/AssignVariableOp_568" input: "^save/AssignVariableOp_569" input: "^save/AssignVariableOp_57" input: "^save/AssignVariableOp_570" input: "^save/AssignVariableOp_571" input: "^save/AssignVariableOp_572" input: "^save/AssignVariableOp_573" input: "^save/AssignVariableOp_574" input: "^save/AssignVariableOp_575" input: "^save/AssignVariableOp_576" input: "^save/AssignVariableOp_577" input: "^save/AssignVariableOp_578" input: "^save/AssignVariableOp_579" input: "^save/AssignVariableOp_58" input: "^save/AssignVariableOp_580" input: "^save/AssignVariableOp_581" input: "^save/AssignVariableOp_582" input: "^save/AssignVariableOp_583" input: "^save/AssignVariableOp_584" input: "^save/AssignVariableOp_585" input: "^save/AssignVariableOp_586" input: "^save/AssignVariableOp_587" input: "^save/AssignVariableOp_588" input: "^save/AssignVariableOp_589" input: "^save/AssignVariableOp_59" input: "^save/AssignVariableOp_590" input: "^save/AssignVariableOp_591" input: "^save/AssignVariableOp_592" input: "^save/AssignVariableOp_593" input: "^save/AssignVariableOp_594" input: "^save/AssignVariableOp_595" input: "^save/AssignVariableOp_596" input: "^save/AssignVariableOp_597" input: "^save/AssignVariableOp_598" input: "^save/AssignVariableOp_599" input: "^save/AssignVariableOp_6" input: "^save/AssignVariableOp_60" input: "^save/AssignVariableOp_600" input: "^save/AssignVariableOp_601" input: "^save/AssignVariableOp_602" input: "^save/AssignVariableOp_603" input: "^save/AssignVariableOp_61" input: "^save/AssignVariableOp_62" input: "^save/AssignVariableOp_63" input: "^save/AssignVariableOp_64" input: "^save/AssignVariableOp_65" input: "^save/AssignVariableOp_66" input: "^save/AssignVariableOp_67" input: "^save/AssignVariableOp_68" input: "^save/AssignVariableOp_69" input: "^save/AssignVariableOp_7" input: "^save/AssignVariableOp_70" input: "^save/AssignVariableOp_71" input: "^save/AssignVariableOp_72" input: "^save/AssignVariableOp_73" input: "^save/AssignVariableOp_74" input: "^save/AssignVariableOp_75" input: "^save/AssignVariableOp_76" input: "^save/AssignVariableOp_77" input: "^save/AssignVariableOp_78" input: "^save/AssignVariableOp_79" input: "^save/AssignVariableOp_8" input: "^save/AssignVariableOp_80" input: "^save/AssignVariableOp_81" input: "^save/AssignVariableOp_82" input: "^save/AssignVariableOp_83" input: "^save/AssignVariableOp_84" input: "^save/AssignVariableOp_85" input: "^save/AssignVariableOp_86" input: "^save/AssignVariableOp_87" input: "^save/AssignVariableOp_88" input: "^save/AssignVariableOp_89" input: "^save/AssignVariableOp_9" input: "^save/AssignVariableOp_90" input: "^save/AssignVariableOp_91" input: "^save/AssignVariableOp_92" input: "^save/AssignVariableOp_93" input: "^save/AssignVariableOp_94" input: "^save/AssignVariableOp_95" input: "^save/AssignVariableOp_96" input: "^save/AssignVariableOp_97" input: "^save/AssignVariableOp_98" input: "^save/AssignVariableOp_99" attr { key: "_has_manual_control_dependencies" value { b: true } } } node { name: "save/restore_all" op: "NoOp" input: "^save/restore_shard" } library { function { signature { name: "__inference_Dataset_flat_map_read_one_file_30" input_arg { name: "args_0" type: DT_STRING } output_arg { name: "identity" type: DT_VARIANT } is_stateful: true control_output: "TFRecordDataset" } node_def { name: "compression_type" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { } string_val: "" } } } } node_def { name: "buffer_size" op: "Const" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { } int64_val: 262144 } } } } node_def { name: "TFRecordDataset" op: "TFRecordDataset" input: "args_0" input: "compression_type:output:0" input: "buffer_size:output:0" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "metadata" value { s: "" } } experimental_type { type_id: TFT_PRODUCT args { type_id: TFT_DATASET args { type_id: TFT_TENSOR args { type_id: TFT_STRING } } } } } node_def { name: "Identity" op: "Identity" input: "TFRecordDataset:handle:0" input: "^NoOp" attr { key: "T" value { type: DT_VARIANT } } attr { key: "_output_shapes" value { list { shape { } } } } } node_def { name: "NoOp" op: "NoOp" input: "^TFRecordDataset" attr { key: "_acd_function_control_output" value { b: true } } attr { key: "_output_shapes" value { list { } } } } ret { key: "identity" value: "Identity:output:0" } attr { key: "_input_shapes" value { list { shape { } } } } attr { key: "_tf_data_function" value { b: true } } control_ret { key: "TFRecordDataset" value: "TFRecordDataset" } arg_attr { value { attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "_user_specified_name" value { s: "args_0" } } } } } function { signature { name: "global_step_cond_true_6" input_arg { name: "read_readvariableop_global_step" type: DT_RESOURCE handle_data { dtype: DT_INT64 shape { } } } input_arg { name: "placeholder" type: DT_INT64 } output_arg { name: "identity" type: DT_INT64 } is_stateful: true } node_def { name: "Read/ReadVariableOp" op: "ReadVariableOp" input: "read_readvariableop_global_step" attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "dtype" value { type: DT_INT64 } } } node_def { name: "Identity" op: "Identity" input: "Read/ReadVariableOp:value:0" attr { key: "T" value { type: DT_INT64 } } attr { key: "_output_shapes" value { list { shape { } } } } } ret { key: "identity" value: "Identity:output:0" } attr { key: "_input_shapes" value { list { shape { } shape { } } } } arg_attr { key: 1 value { attr { key: "_output_shapes" value { list { shape { } } } } } } } function { signature { name: "global_step_cond_false_7" input_arg { name: "placeholder" type: DT_RESOURCE } input_arg { name: "global_step_initializer_zeros_0" type: DT_INT64 } output_arg { name: "global_step_initializer_zeros" type: DT_INT64 } } ret { key: "global_step_initializer_zeros" value: "global_step_initializer_zeros_0" } attr { key: "_input_shapes" value { list { shape { } shape { } } } } arg_attr { key: 1 value { attr { key: "_output_shapes" value { list { shape { } } } } } } } function { signature { name: "__inference_tf_data_experimental_map_and_batch_lambda_64" input_arg { name: "args_0" type: DT_STRING } output_arg { name: "identity" type: DT_INT32 } output_arg { name: "identity_1" type: DT_INT32 } output_arg { name: "identity_2" type: DT_INT32 } output_arg { name: "identity_3" type: DT_INT32 } output_arg { name: "identity_4" type: DT_INT32 } } node_def { name: "ParseSingleExample/ParseExample/Const" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { dim { } } } } } } node_def { name: "ParseSingleExample/ParseExample/Const_1" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { dim { } } } } } } node_def { name: "ParseSingleExample/ParseExample/Const_2" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { dim { } } } } } } node_def { name: "ParseSingleExample/ParseExample/Const_3" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { dim { } } } } } } node_def { name: "ParseSingleExample/ParseExample/Const_4" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_INT64 } } attr { key: "value" value { tensor { dtype: DT_INT64 tensor_shape { dim { } } } } } } node_def { name: "ParseSingleExample/ParseExample/ParseExampleV2/names" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { } } } } } } node_def { name: "ParseSingleExample/ParseExample/ParseExampleV2/sparse_keys" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { } } } } } } node_def { name: "ParseSingleExample/ParseExample/ParseExampleV2/dense_keys" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { size: 5 } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { size: 5 } } string_val: "input_ids" string_val: "input_mask" string_val: "is_real_example" string_val: "label_ids" string_val: "segment_ids" } } } } node_def { name: "ParseSingleExample/ParseExample/ParseExampleV2/ragged_keys" op: "Const" attr { key: "_output_shapes" value { list { shape { dim { } } } } } attr { key: "dtype" value { type: DT_STRING } } attr { key: "value" value { tensor { dtype: DT_STRING tensor_shape { dim { } } } } } } node_def { name: "ParseSingleExample/ParseExample/ParseExampleV2" op: "ParseExampleV2" input: "args_0" input: "ParseSingleExample/ParseExample/ParseExampleV2/names:output:0" input: "ParseSingleExample/ParseExample/ParseExampleV2/sparse_keys:output:0" input: "ParseSingleExample/ParseExample/ParseExampleV2/dense_keys:output:0" input: "ParseSingleExample/ParseExample/ParseExampleV2/ragged_keys:output:0" input: "ParseSingleExample/ParseExample/Const:output:0" input: "ParseSingleExample/ParseExample/Const_1:output:0" input: "ParseSingleExample/ParseExample/Const_2:output:0" input: "ParseSingleExample/ParseExample/Const_3:output:0" input: "ParseSingleExample/ParseExample/Const_4:output:0" attr { key: "Tdense" value { list { type: DT_INT64 type: DT_INT64 type: DT_INT64 type: DT_INT64 type: DT_INT64 } } } attr { key: "_output_shapes" value { list { shape { dim { size: 300 } } shape { dim { size: 300 } } shape { } shape { } shape { dim { size: 300 } } } } } attr { key: "dense_shapes" value { list { shape { dim { size: 300 } } shape { dim { size: 300 } } shape { } shape { } shape { dim { size: 300 } } } } } attr { key: "num_sparse" value { i: 0 } } attr { key: "ragged_split_types" value { list { } } } attr { key: "ragged_value_types" value { list { } } } attr { key: "sparse_types" value { list { } } } } node_def { name: "ToInt32" op: "Cast" input: "ParseSingleExample/ParseExample/ParseExampleV2:dense_values:0" attr { key: "DstT" value { type: DT_INT32 } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { dim { size: 300 } } } } } } node_def { name: "ToInt32_1" op: "Cast" input: "ParseSingleExample/ParseExample/ParseExampleV2:dense_values:1" attr { key: "DstT" value { type: DT_INT32 } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { dim { size: 300 } } } } } } node_def { name: "ToInt32_2" op: "Cast" input: "ParseSingleExample/ParseExample/ParseExampleV2:dense_values:2" attr { key: "DstT" value { type: DT_INT32 } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { } } } } } node_def { name: "ToInt32_3" op: "Cast" input: "ParseSingleExample/ParseExample/ParseExampleV2:dense_values:3" attr { key: "DstT" value { type: DT_INT32 } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { } } } } } node_def { name: "ToInt32_4" op: "Cast" input: "ParseSingleExample/ParseExample/ParseExampleV2:dense_values:4" attr { key: "DstT" value { type: DT_INT32 } } attr { key: "SrcT" value { type: DT_INT64 } } attr { key: "Truncate" value { b: false } } attr { key: "_output_shapes" value { list { shape { dim { size: 300 } } } } } } node_def { name: "Identity" op: "Identity" input: "ToInt32:y:0" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 300 } } } } } } node_def { name: "Identity_1" op: "Identity" input: "ToInt32_1:y:0" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 300 } } } } } } node_def { name: "Identity_2" op: "Identity" input: "ToInt32_2:y:0" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node_def { name: "Identity_3" op: "Identity" input: "ToInt32_3:y:0" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { } } } } } node_def { name: "Identity_4" op: "Identity" input: "ToInt32_4:y:0" attr { key: "T" value { type: DT_INT32 } } attr { key: "_output_shapes" value { list { shape { dim { size: 300 } } } } } } ret { key: "identity" value: "Identity:output:0" } ret { key: "identity_1" value: "Identity_1:output:0" } ret { key: "identity_2" value: "Identity_2:output:0" } ret { key: "identity_3" value: "Identity_3:output:0" } ret { key: "identity_4" value: "Identity_4:output:0" } attr { key: "_input_shapes" value { list { shape { } } } } attr { key: "_tf_data_function" value { b: true } } arg_attr { value { attr { key: "_output_shapes" value { list { shape { } } } } attr { key: "_user_specified_name" value { s: "args_0" } } } } } } versions { producer: 1395 min_consumer: 12 }