修改部分配置文件

roformer_chinese_sim_char_base/README.md

@@ -0,0 +1,4 @@
+# 说明
+
+- 可直接下载第三方用户转好的[pytorch权重](https://huggingface.co/junnyu/roformer_chinese_sim_char_base)
+- 也可下载[tf权重](https://github.com/ZhuiyiTechnology/roformer-sim), 并使用convert.py脚本转换

roformer_chinese_sim_char_base/convert.py

@@ -0,0 +1,69 @@
+# roformer-sim(simbert v2)预训练模型tensorflow转pytorch
+# 源项目：https://github.com/ZhuiyiTechnology/roformer-sim
+# 跟simbert(v1)最主要的不同是不需要加载位置编码的部分,苏神ckpt也同样无该部分,加载进模型后使用rope位置编码
+
+import torch
+import tensorflow as tf
+import json
+
+tf_dir = 'E:/pretrain_ckpt/simbert/sushen@chinese_roformer-sim-char_L-12_H-768_A-12/'
+tf_path = tf_dir + 'bert_model.ckpt'
+torch_path = 'E:/pretrain_ckpt/simbert/sushen@chinese_roformer-sim-char_L-12_H-768_A-12/pytorch_model.bin'
+
+with open(tf_dir + 'bert_config.json', 'r') as f:
+    config = json.load(f)
+    num_layers = config['num_hidden_layers']
+
+torch_state_dict = {}
+
+prefix = 'roformer'
+mapping = {
+'bert/embeddings/word_embeddings':  f'{prefix}.embeddings.word_embeddings.weight',
+'bert/embeddings/token_type_embeddings': f'{prefix}.embeddings.token_type_embeddings.weight',
+'bert/embeddings/LayerNorm/beta': f'{prefix}.embeddings.LayerNorm.bias',
+'bert/embeddings/LayerNorm/gamma': f'{prefix}.embeddings.LayerNorm.weight',
+'cls/predictions/transform/dense/kernel': 'cls.predictions.transform.dense.weight##',
+'cls/predictions/transform/dense/bias': 'cls.predictions.transform.dense.bias',
+'cls/predictions/transform/LayerNorm/beta': 'cls.predictions.transform.LayerNorm.bias',
+'cls/predictions/transform/LayerNorm/gamma': 'cls.predictions.transform.LayerNorm.weight',
+'cls/predictions/output_bias': 'cls.predictions.bias',
+'bert/pooler/dense/kernel': f'{prefix}.pooler.dense.weight##',
+'bert/pooler/dense/bias': f'{prefix}.pooler.dense.bias'}
+
+if ('embedding_size' in config) and (config['embedding_size'] != config['hidden_size']):
+    mapping.update({'bert/encoder/embedding_hidden_mapping_in/kernel': f'{prefix}.encoder.embedding_hidden_mapping_in.weight##',
+    'bert/encoder/embedding_hidden_mapping_in/bias': f'{prefix}.encoder.embedding_hidden_mapping_in.bias'})
+
+for i in range(num_layers):
+    prefix_i = f'{prefix}.encoder.layer.%d.' % i
+    mapping.update({
+        f'bert/encoder/layer_{i}/attention/self/query/kernel': prefix_i + 'attention.self.query.weight##',  # 转置标识
+        f'bert/encoder/layer_{i}/attention/self/query/bias': prefix_i + 'attention.self.query.bias',
+        f'bert/encoder/layer_{i}/attention/self/key/kernel': prefix_i + 'attention.self.key.weight##',
+        f'bert/encoder/layer_{i}/attention/self/key/bias': prefix_i + 'attention.self.key.bias',
+        f'bert/encoder/layer_{i}/attention/self/value/kernel': prefix_i + 'attention.self.value.weight##',
+        f'bert/encoder/layer_{i}/attention/self/value/bias': prefix_i + 'attention.self.value.bias',
+        f'bert/encoder/layer_{i}/attention/output/dense/kernel': prefix_i + 'attention.output.dense.weight##',
+        f'bert/encoder/layer_{i}/attention/output/dense/bias': prefix_i + 'attention.output.dense.bias',
+        f'bert/encoder/layer_{i}/attention/output/LayerNorm/beta': prefix_i + 'attention.output.LayerNorm.bias',
+        f'bert/encoder/layer_{i}/attention/output/LayerNorm/gamma': prefix_i + 'attention.output.LayerNorm.weight',
+        f'bert/encoder/layer_{i}/intermediate/dense/kernel': prefix_i + 'intermediate.dense.weight##',
+        f'bert/encoder/layer_{i}/intermediate/dense/bias': prefix_i + 'intermediate.dense.bias',
+        f'bert/encoder/layer_{i}/output/dense/kernel': prefix_i + 'output.dense.weight##',
+        f'bert/encoder/layer_{i}/output/dense/bias': prefix_i + 'output.dense.bias',
+        f'bert/encoder/layer_{i}/output/LayerNorm/beta': prefix_i + 'output.LayerNorm.bias',
+        f'bert/encoder/layer_{i}/output/LayerNorm/gamma': prefix_i + 'output.LayerNorm.weight'
+    })
+
+
+for key, value in mapping.items():
+    ts = tf.train.load_variable(tf_path, key)
+    if value.endswith('##'):
+        value = value.replace('##', '')
+        torch_state_dict[value] = torch.from_numpy(ts).T
+    else:
+        torch_state_dict[value] = torch.from_numpy(ts)
+torch_state_dict['cls.predictions.decoder.weight'] = torch_state_dict[f'{prefix}.embeddings.word_embeddings.weight']
+torch_state_dict['cls.predictions.decoder.bias'] = torch_state_dict['cls.predictions.bias']
+
+torch.save(torch_state_dict, torch_path)

roformer_chinese_sim_char_ft_base/README.md

@@ -0,0 +1,4 @@
+# 说明
+
+- 可直接下载第三方用户转好的[pytorch权重](https://huggingface.co/junnyu/roformer_chinese_sim_char_ft_base)
+- 也可下载[tf权重](https://github.com/ZhuiyiTechnology/roformer-sim), 并使用convert.py脚本转换

roformer_chinese_sim_char_ft_base/convert.py

@@ -0,0 +1,69 @@
+# roformer-sim(simbert v2)预训练模型tensorflow转pytorch
+# 源项目：https://github.com/ZhuiyiTechnology/roformer-sim
+# 跟simbert(v1)最主要的不同是不需要加载位置编码的部分,苏神ckpt也同样无该部分,加载进模型后使用rope位置编码
+
+import torch
+import tensorflow as tf
+import json
+
+tf_dir = 'E:/pretrain_ckpt/simbert/sushen@chinese_roformer-sim-char_L-12_H-768_A-12/'
+tf_path = tf_dir + 'bert_model.ckpt'
+torch_path = 'E:/pretrain_ckpt/simbert/sushen@chinese_roformer-sim-char_L-12_H-768_A-12/pytorch_model.bin'
+
+with open(tf_dir + 'bert_config.json', 'r') as f:
+    config = json.load(f)
+    num_layers = config['num_hidden_layers']
+
+torch_state_dict = {}
+
+prefix = 'roformer'
+mapping = {
+'bert/embeddings/word_embeddings':  f'{prefix}.embeddings.word_embeddings.weight',
+'bert/embeddings/token_type_embeddings': f'{prefix}.embeddings.token_type_embeddings.weight',
+'bert/embeddings/LayerNorm/beta': f'{prefix}.embeddings.LayerNorm.bias',
+'bert/embeddings/LayerNorm/gamma': f'{prefix}.embeddings.LayerNorm.weight',
+'cls/predictions/transform/dense/kernel': 'cls.predictions.transform.dense.weight##',
+'cls/predictions/transform/dense/bias': 'cls.predictions.transform.dense.bias',
+'cls/predictions/transform/LayerNorm/beta': 'cls.predictions.transform.LayerNorm.bias',
+'cls/predictions/transform/LayerNorm/gamma': 'cls.predictions.transform.LayerNorm.weight',
+'cls/predictions/output_bias': 'cls.predictions.bias',
+'bert/pooler/dense/kernel': f'{prefix}.pooler.dense.weight##',
+'bert/pooler/dense/bias': f'{prefix}.pooler.dense.bias'}
+
+if ('embedding_size' in config) and (config['embedding_size'] != config['hidden_size']):
+    mapping.update({'bert/encoder/embedding_hidden_mapping_in/kernel': f'{prefix}.encoder.embedding_hidden_mapping_in.weight##',
+    'bert/encoder/embedding_hidden_mapping_in/bias': f'{prefix}.encoder.embedding_hidden_mapping_in.bias'})
+
+for i in range(num_layers):
+    prefix_i = f'{prefix}.encoder.layer.%d.' % i
+    mapping.update({
+        f'bert/encoder/layer_{i}/attention/self/query/kernel': prefix_i + 'attention.self.query.weight##',  # 转置标识
+        f'bert/encoder/layer_{i}/attention/self/query/bias': prefix_i + 'attention.self.query.bias',
+        f'bert/encoder/layer_{i}/attention/self/key/kernel': prefix_i + 'attention.self.key.weight##',
+        f'bert/encoder/layer_{i}/attention/self/key/bias': prefix_i + 'attention.self.key.bias',
+        f'bert/encoder/layer_{i}/attention/self/value/kernel': prefix_i + 'attention.self.value.weight##',
+        f'bert/encoder/layer_{i}/attention/self/value/bias': prefix_i + 'attention.self.value.bias',
+        f'bert/encoder/layer_{i}/attention/output/dense/kernel': prefix_i + 'attention.output.dense.weight##',
+        f'bert/encoder/layer_{i}/attention/output/dense/bias': prefix_i + 'attention.output.dense.bias',
+        f'bert/encoder/layer_{i}/attention/output/LayerNorm/beta': prefix_i + 'attention.output.LayerNorm.bias',
+        f'bert/encoder/layer_{i}/attention/output/LayerNorm/gamma': prefix_i + 'attention.output.LayerNorm.weight',
+        f'bert/encoder/layer_{i}/intermediate/dense/kernel': prefix_i + 'intermediate.dense.weight##',
+        f'bert/encoder/layer_{i}/intermediate/dense/bias': prefix_i + 'intermediate.dense.bias',
+        f'bert/encoder/layer_{i}/output/dense/kernel': prefix_i + 'output.dense.weight##',
+        f'bert/encoder/layer_{i}/output/dense/bias': prefix_i + 'output.dense.bias',
+        f'bert/encoder/layer_{i}/output/LayerNorm/beta': prefix_i + 'output.LayerNorm.bias',
+        f'bert/encoder/layer_{i}/output/LayerNorm/gamma': prefix_i + 'output.LayerNorm.weight'
+    })
+
+
+for key, value in mapping.items():
+    ts = tf.train.load_variable(tf_path, key)
+    if value.endswith('##'):
+        value = value.replace('##', '')
+        torch_state_dict[value] = torch.from_numpy(ts).T
+    else:
+        torch_state_dict[value] = torch.from_numpy(ts)
+torch_state_dict['cls.predictions.decoder.weight'] = torch_state_dict[f'{prefix}.embeddings.word_embeddings.weight']
+torch_state_dict['cls.predictions.decoder.bias'] = torch_state_dict['cls.predictions.bias']
+
+torch.save(torch_state_dict, torch_path)

roformer_chinese_sim_char_ft_small/README.md

@@ -0,0 +1,4 @@
+# 说明
+
+- 可直接下载第三方用户转好的[pytorch权重](https://huggingface.co/junnyu/roformer_chinese_sim_char_ft_small)
+- 也可下载[tf权重](https://github.com/ZhuiyiTechnology/roformer-sim), 并使用convert.py脚本转换

roformer_chinese_sim_char_ft_small/convert.py

@@ -0,0 +1,69 @@
+# roformer-sim(simbert v2)预训练模型tensorflow转pytorch
+# 源项目：https://github.com/ZhuiyiTechnology/roformer-sim
+# 跟simbert(v1)最主要的不同是不需要加载位置编码的部分,苏神ckpt也同样无该部分,加载进模型后使用rope位置编码
+
+import torch
+import tensorflow as tf
+import json
+
+tf_dir = 'E:/pretrain_ckpt/simbert/sushen@chinese_roformer-sim-char_L-12_H-768_A-12/'
+tf_path = tf_dir + 'bert_model.ckpt'
+torch_path = 'E:/pretrain_ckpt/simbert/sushen@chinese_roformer-sim-char_L-12_H-768_A-12/pytorch_model.bin'
+
+with open(tf_dir + 'bert_config.json', 'r') as f:
+    config = json.load(f)
+    num_layers = config['num_hidden_layers']
+
+torch_state_dict = {}
+
+prefix = 'roformer'
+mapping = {
+'bert/embeddings/word_embeddings':  f'{prefix}.embeddings.word_embeddings.weight',
+'bert/embeddings/token_type_embeddings': f'{prefix}.embeddings.token_type_embeddings.weight',
+'bert/embeddings/LayerNorm/beta': f'{prefix}.embeddings.LayerNorm.bias',
+'bert/embeddings/LayerNorm/gamma': f'{prefix}.embeddings.LayerNorm.weight',
+'cls/predictions/transform/dense/kernel': 'cls.predictions.transform.dense.weight##',
+'cls/predictions/transform/dense/bias': 'cls.predictions.transform.dense.bias',
+'cls/predictions/transform/LayerNorm/beta': 'cls.predictions.transform.LayerNorm.bias',
+'cls/predictions/transform/LayerNorm/gamma': 'cls.predictions.transform.LayerNorm.weight',
+'cls/predictions/output_bias': 'cls.predictions.bias',
+'bert/pooler/dense/kernel': f'{prefix}.pooler.dense.weight##',
+'bert/pooler/dense/bias': f'{prefix}.pooler.dense.bias'}
+
+if ('embedding_size' in config) and (config['embedding_size'] != config['hidden_size']):
+    mapping.update({'bert/encoder/embedding_hidden_mapping_in/kernel': f'{prefix}.encoder.embedding_hidden_mapping_in.weight##',
+    'bert/encoder/embedding_hidden_mapping_in/bias': f'{prefix}.encoder.embedding_hidden_mapping_in.bias'})
+
+for i in range(num_layers):
+    prefix_i = f'{prefix}.encoder.layer.%d.' % i
+    mapping.update({
+        f'bert/encoder/layer_{i}/attention/self/query/kernel': prefix_i + 'attention.self.query.weight##',  # 转置标识
+        f'bert/encoder/layer_{i}/attention/self/query/bias': prefix_i + 'attention.self.query.bias',
+        f'bert/encoder/layer_{i}/attention/self/key/kernel': prefix_i + 'attention.self.key.weight##',
+        f'bert/encoder/layer_{i}/attention/self/key/bias': prefix_i + 'attention.self.key.bias',
+        f'bert/encoder/layer_{i}/attention/self/value/kernel': prefix_i + 'attention.self.value.weight##',
+        f'bert/encoder/layer_{i}/attention/self/value/bias': prefix_i + 'attention.self.value.bias',
+        f'bert/encoder/layer_{i}/attention/output/dense/kernel': prefix_i + 'attention.output.dense.weight##',
+        f'bert/encoder/layer_{i}/attention/output/dense/bias': prefix_i + 'attention.output.dense.bias',
+        f'bert/encoder/layer_{i}/attention/output/LayerNorm/beta': prefix_i + 'attention.output.LayerNorm.bias',
+        f'bert/encoder/layer_{i}/attention/output/LayerNorm/gamma': prefix_i + 'attention.output.LayerNorm.weight',
+        f'bert/encoder/layer_{i}/intermediate/dense/kernel': prefix_i + 'intermediate.dense.weight##',
+        f'bert/encoder/layer_{i}/intermediate/dense/bias': prefix_i + 'intermediate.dense.bias',
+        f'bert/encoder/layer_{i}/output/dense/kernel': prefix_i + 'output.dense.weight##',
+        f'bert/encoder/layer_{i}/output/dense/bias': prefix_i + 'output.dense.bias',
+        f'bert/encoder/layer_{i}/output/LayerNorm/beta': prefix_i + 'output.LayerNorm.bias',
+        f'bert/encoder/layer_{i}/output/LayerNorm/gamma': prefix_i + 'output.LayerNorm.weight'
+    })
+
+
+for key, value in mapping.items():
+    ts = tf.train.load_variable(tf_path, key)
+    if value.endswith('##'):
+        value = value.replace('##', '')
+        torch_state_dict[value] = torch.from_numpy(ts).T
+    else:
+        torch_state_dict[value] = torch.from_numpy(ts)
+torch_state_dict['cls.predictions.decoder.weight'] = torch_state_dict[f'{prefix}.embeddings.word_embeddings.weight']
+torch_state_dict['cls.predictions.decoder.bias'] = torch_state_dict['cls.predictions.bias']
+
+torch.save(torch_state_dict, torch_path)

roformer_chinese_sim_char_small/README.md

@@ -0,0 +1,4 @@
+# 说明
+
+- 可直接下载第三方用户转好的[pytorch权重](https://huggingface.co/junnyu/roformer_chinese_sim_char_small)
+- 也可下载[tf权重](https://github.com/ZhuiyiTechnology/roformer-sim), 并使用convert.py脚本转换

roformer_chinese_sim_char_small/convert.py

@@ -0,0 +1,69 @@
+# roformer-sim(simbert v2)预训练模型tensorflow转pytorch
+# 源项目：https://github.com/ZhuiyiTechnology/roformer-sim
+# 跟simbert(v1)最主要的不同是不需要加载位置编码的部分,苏神ckpt也同样无该部分,加载进模型后使用rope位置编码
+
+import torch
+import tensorflow as tf
+import json
+
+tf_dir = 'E:/pretrain_ckpt/simbert/sushen@chinese_roformer-sim-char_L-12_H-768_A-12/'
+tf_path = tf_dir + 'bert_model.ckpt'
+torch_path = 'E:/pretrain_ckpt/simbert/sushen@chinese_roformer-sim-char_L-12_H-768_A-12/pytorch_model.bin'
+
+with open(tf_dir + 'bert_config.json', 'r') as f:
+    config = json.load(f)
+    num_layers = config['num_hidden_layers']
+
+torch_state_dict = {}
+
+prefix = 'roformer'
+mapping = {
+'bert/embeddings/word_embeddings':  f'{prefix}.embeddings.word_embeddings.weight',
+'bert/embeddings/token_type_embeddings': f'{prefix}.embeddings.token_type_embeddings.weight',
+'bert/embeddings/LayerNorm/beta': f'{prefix}.embeddings.LayerNorm.bias',
+'bert/embeddings/LayerNorm/gamma': f'{prefix}.embeddings.LayerNorm.weight',
+'cls/predictions/transform/dense/kernel': 'cls.predictions.transform.dense.weight##',
+'cls/predictions/transform/dense/bias': 'cls.predictions.transform.dense.bias',
+'cls/predictions/transform/LayerNorm/beta': 'cls.predictions.transform.LayerNorm.bias',
+'cls/predictions/transform/LayerNorm/gamma': 'cls.predictions.transform.LayerNorm.weight',
+'cls/predictions/output_bias': 'cls.predictions.bias',
+'bert/pooler/dense/kernel': f'{prefix}.pooler.dense.weight##',
+'bert/pooler/dense/bias': f'{prefix}.pooler.dense.bias'}
+
+if ('embedding_size' in config) and (config['embedding_size'] != config['hidden_size']):
+    mapping.update({'bert/encoder/embedding_hidden_mapping_in/kernel': f'{prefix}.encoder.embedding_hidden_mapping_in.weight##',
+    'bert/encoder/embedding_hidden_mapping_in/bias': f'{prefix}.encoder.embedding_hidden_mapping_in.bias'})
+
+for i in range(num_layers):
+    prefix_i = f'{prefix}.encoder.layer.%d.' % i
+    mapping.update({
+        f'bert/encoder/layer_{i}/attention/self/query/kernel': prefix_i + 'attention.self.query.weight##',  # 转置标识
+        f'bert/encoder/layer_{i}/attention/self/query/bias': prefix_i + 'attention.self.query.bias',
+        f'bert/encoder/layer_{i}/attention/self/key/kernel': prefix_i + 'attention.self.key.weight##',
+        f'bert/encoder/layer_{i}/attention/self/key/bias': prefix_i + 'attention.self.key.bias',
+        f'bert/encoder/layer_{i}/attention/self/value/kernel': prefix_i + 'attention.self.value.weight##',
+        f'bert/encoder/layer_{i}/attention/self/value/bias': prefix_i + 'attention.self.value.bias',
+        f'bert/encoder/layer_{i}/attention/output/dense/kernel': prefix_i + 'attention.output.dense.weight##',
+        f'bert/encoder/layer_{i}/attention/output/dense/bias': prefix_i + 'attention.output.dense.bias',
+        f'bert/encoder/layer_{i}/attention/output/LayerNorm/beta': prefix_i + 'attention.output.LayerNorm.bias',
+        f'bert/encoder/layer_{i}/attention/output/LayerNorm/gamma': prefix_i + 'attention.output.LayerNorm.weight',
+        f'bert/encoder/layer_{i}/intermediate/dense/kernel': prefix_i + 'intermediate.dense.weight##',
+        f'bert/encoder/layer_{i}/intermediate/dense/bias': prefix_i + 'intermediate.dense.bias',
+        f'bert/encoder/layer_{i}/output/dense/kernel': prefix_i + 'output.dense.weight##',
+        f'bert/encoder/layer_{i}/output/dense/bias': prefix_i + 'output.dense.bias',
+        f'bert/encoder/layer_{i}/output/LayerNorm/beta': prefix_i + 'output.LayerNorm.bias',
+        f'bert/encoder/layer_{i}/output/LayerNorm/gamma': prefix_i + 'output.LayerNorm.weight'
+    })
+
+
+for key, value in mapping.items():
+    ts = tf.train.load_variable(tf_path, key)
+    if value.endswith('##'):
+        value = value.replace('##', '')
+        torch_state_dict[value] = torch.from_numpy(ts).T
+    else:
+        torch_state_dict[value] = torch.from_numpy(ts)
+torch_state_dict['cls.predictions.decoder.weight'] = torch_state_dict[f'{prefix}.embeddings.word_embeddings.weight']
+torch_state_dict['cls.predictions.decoder.bias'] = torch_state_dict['cls.predictions.bias']
+
+torch.save(torch_state_dict, torch_path)

simbert-chinese-base/bert4torch_config.json

@@ -1,22 +0,0 @@
-{
-  "attention_probs_dropout_prob": 0.1, 
-  "directionality": "bidi", 
-  "hidden_act": "gelu", 
-  "hidden_dropout_prob": 0.1, 
-  "hidden_size": 768, 
-  "initializer_range": 0.02, 
-  "intermediate_size": 3072, 
-  "max_position_embeddings": 512, 
-  "model_type": "bert",
-  "num_attention_heads": 12, 
-  "num_hidden_layers": 12, 
-  "pooler_fc_size": 768, 
-  "pooler_num_attention_heads": 12, 
-  "pooler_num_fc_layers": 3, 
-  "pooler_size_per_head": 128, 
-  "pooler_type": "first_token_transform", 
-  "type_vocab_size": 2, 
-  "vocab_size": 13685,
-  "with_pool": "linear",
-  "pool_strategy": "pooler"
-}

simbert_chinese_small/bert4torch_config.json

@@ -1,17 +0,0 @@
-{
-  "attention_probs_dropout_prob": 0.0,
-  "directionality": "bidi",
-  "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.0,
-  "hidden_size": 384,
-  "embedding_size": 128,
-  "initializer_range": 0.02,
-  "intermediate_size": 1536,
-  "max_position_embeddings": 512,
-  "num_attention_heads": 12,
-  "num_hidden_layers": 6,
-  "type_vocab_size": 2,
-  "vocab_size": 13685,
-  "with_pool": "linear",
-  "pool_strategy": "pooler"
-}

simbert_chinese_tiny/bert4torch_config.json

@@ -1,17 +0,0 @@
-{
-  "attention_probs_dropout_prob": 0.0,
-  "directionality": "bidi",
-  "hidden_act": "gelu",
-  "hidden_dropout_prob": 0.0,
-  "hidden_size": 312,
-  "embedding_size": 128,
-  "initializer_range": 0.02,
-  "intermediate_size": 1248,
-  "max_position_embeddings": 512,
-  "num_attention_heads": 12,
-  "num_hidden_layers": 4,
-  "type_vocab_size": 2,
-  "vocab_size": 13685,
-  "with_pool": "linear",
-  "pool_strategy": "pooler"
-}