[20230821121100]

.gitignore

@@ -0,0 +1,6 @@
+
+.git/
+.idea/
+
+**/flagged/
+**/__pycache__/

README.md

@@ -4,8 +4,8 @@ emoji: 🐠
 colorFrom: red
 colorTo: purple
 sdk: gradio
-sdk_version: 3.40.1
-app_file: app.py
+sdk_version: 3.20.1
+app_file: main.py
 pinned: false
 ---
 

examples/exercises/firefly_bloom_1b4/1.train_model.py

@@ -0,0 +1,150 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import os
+import platform
+
+os.environ["CUDA_LAUNCH_BLOCKING"] = "1"
+
+from datasets import Dataset, DatasetDict, load_dataset
+from transformers.data.data_collator import DataCollatorForLanguageModeling
+from transformers import BloomTokenizerFast, BloomForCausalLM
+from transformers.trainer import Trainer
+from transformers.training_args import TrainingArguments
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--train_file",
+        # default='firefly-train-1.1M.jsonl',
+        default="D:/programmer/nlp_datasets/firefly-train-1.1M.jsonl",
+        type=str
+    )
+    parser.add_argument(
+        "--pretrained_model_name_or_path",
+        # default='YeungNLP/bloom-1b4-zh',
+        default="D:/programmer/nlp_pretrained_model/bloom-1b7",
+        type=str,
+    )
+    parser.add_argument("--cache_dir", default="cache_dir", type=str)
+
+    parser.add_argument("--output_dir", default="serialization_dir", type=str)
+    parser.add_argument("--overwrite_output_dir", action="store_true")
+    parser.add_argument("--evaluation_strategy", default="no", choices=["no", "steps", "epoch"], type=str)
+    parser.add_argument("--per_device_train_batch_size", default=4, type=int)
+    parser.add_argument("--gradient_accumulation_steps", default=4, type=int)
+    parser.add_argument("--learning_rate", default=1e-5, type=float)
+    parser.add_argument("--weight_decay", default=0, type=float)
+    parser.add_argument("--max_grad_norm", default=1.0, type=float)
+    parser.add_argument("--num_train_epochs", default=3.0, type=float)
+    parser.add_argument("--max_steps", default=-1, type=int)
+    parser.add_argument("--lr_scheduler_type", default="cosine", type=str)
+    parser.add_argument("--warmup_ratio", default=0.0, type=float)
+    parser.add_argument("--warmup_steps", default=3000, type=int)
+    parser.add_argument("--logging_steps", default=300, type=int)
+    parser.add_argument("--save_strategy", default="steps", type=str)
+    parser.add_argument("--save_steps", default=500, type=int)
+    parser.add_argument("--save_total_limit", default=3, type=int)
+    parser.add_argument("--no_cuda", action="store_true")
+    parser.add_argument("--seed", default=3407, type=str, help="https://arxiv.org/abs/2109.08203")
+    parser.add_argument("--fp16", action="store_true")
+    parser.add_argument("--half_precision_backend", default="auto", type=str)
+    parser.add_argument("--dataloader_num_workers", default=5, type=int)
+    parser.add_argument("--disable_tqdm", action="store_false")
+    parser.add_argument("--remove_unused_columns", action="store_false")
+    # parser.add_argument("--deepspeed", default="ds_z3_config.json", type=str)
+    parser.add_argument("--deepspeed", default=None, type=str)
+    parser.add_argument("--optim", default="adamw_hf", type=str)
+    parser.add_argument("--report_to", default="tensorboard", type=str)
+    parser.add_argument("--resume_from_checkpoint", default="file_dir/serialization_dir/checkpoint-103000", type=str)
+    parser.add_argument("--gradient_checkpointing", action="store_true")
+    # parser.add_argument("--gradient_checkpointing", action="store_false")
+
+    parser.add_argument("--truncate_longer_samples", action="store_true")
+    parser.add_argument("--max_seq_length", default=512, type=int)
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    os.makedirs(args.output_dir, exist_ok=True)
+    os.makedirs(args.cache_dir, exist_ok=True)
+
+    # dataset
+    dataset_dict = DatasetDict()
+    train_data_files = [args.train_file]
+    dataset_dict["train"] = load_dataset(
+        path="json", data_files=[str(file) for file in train_data_files]
+    )["train"]
+    print(dataset_dict)
+
+    # pretrained model
+    tokenizer = BloomTokenizerFast.from_pretrained(args.pretrained_model_name_or_path)
+    model = BloomForCausalLM.from_pretrained(args.pretrained_model_name_or_path)
+
+    def encode_with_truncation(examples):
+        input_ = examples.pop("input")
+        target_ = examples.pop("target")
+        text = "<s>{input}</s></s>{target}</s>".format(input=input_, target=target_)
+        result = tokenizer(
+            text,
+            truncation=True,
+            # padding='max_length',
+            max_length=args.max_seq_length,
+            return_special_tokens_mask=True
+        )
+        return result
+
+    train_dataset = dataset_dict["train"].map(
+        encode_with_truncation,
+        batched=False,
+        keep_in_memory=False,
+        num_proc=None if platform.system() == "Windows" else os.cpu_count(),
+        cache_file_name=os.path.join(args.cache_dir, "train.cache")
+    )
+    train_dataset.set_format(type="torch", columns=["input_ids", "attention_mask"])
+
+    print("Train Dataset Examples Batch Number: {}".format(len(train_dataset)))
+
+    # training
+    data_collator = DataCollatorForLanguageModeling(
+        tokenizer=tokenizer, mlm=False
+    )
+    training_args = TrainingArguments(
+        output_dir=args.output_dir,
+        overwrite_output_dir=args.overwrite_output_dir,
+        evaluation_strategy=args.evaluation_strategy,
+        per_device_train_batch_size=args.per_device_train_batch_size,
+        gradient_accumulation_steps=args.gradient_accumulation_steps,
+        learning_rate=args.learning_rate,
+        num_train_epochs=args.num_train_epochs,
+        max_steps=args.max_steps,
+        lr_scheduler_type=args.lr_scheduler_type,
+        warmup_steps=args.warmup_steps,
+        logging_steps=args.logging_steps,
+        save_steps=args.save_steps,
+        save_total_limit=args.save_total_limit,
+        no_cuda=args.no_cuda,
+        fp16=args.fp16,
+        half_precision_backend=args.half_precision_backend,
+        # deepspeed=args.deepspeed,
+        report_to=args.report_to,
+        resume_from_checkpoint=args.resume_from_checkpoint,
+        gradient_checkpointing=args.gradient_checkpointing,
+    )
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        data_collator=data_collator,
+        train_dataset=train_dataset,
+    )
+    trainer.train()
+    return
+
+
+if __name__ == '__main__':
+    main()

examples/exercises/firefly_bloom_1b4/2.test_sft_model.py

@@ -0,0 +1,81 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import os
+import sys
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, '../../../'))
+
+import torch
+from transformers import BloomTokenizerFast, BloomForCausalLM
+
+from project_settings import project_path
+
+
+def get_args():
+    """
+    python3 2.test_sft_model.py --trained_model_path /data/tianxing/PycharmProjects/Transformers/trained_models/bloom-396m-sft
+    python3 2.test_sft_model.py --trained_model_path /data/tianxing/PycharmProjects/Transformers/trained_models/bloom-1b4-sft
+
+    参考链接:
+    https://huggingface.co/YeungNLP/firefly-bloom-1b4
+
+    Example:
+        将下面句子翻译成现代文：\n石中央又生一树，高百余尺，条干偃阴为五色，翠叶如盘，花径尺余，色深碧，蕊深红，异香成烟，著物霏霏。
+
+        实体识别: 1949年10月1日，人们在北京天安门广场参加开国大典。
+
+        把这句话翻译成英文: 1949年10月1日，人们在北京天安门广场参加开国大典。
+
+        晚上睡不着该怎么办. 请给点详细的介绍.
+
+        将下面的句子翻译成文言文：结婚率下降, 离婚率暴增, 生育率下降, 人民焦虑迷茫, 到底是谁的错.
+
+        对联：厌烟沿檐烟燕眼. (污雾舞坞寤梧芜).
+
+        写一首咏雪的古诗, 标题为 "沁园春, 雪".
+
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--trained_model_path',
+        # default='YeungNLP/bloom-1b4-zh',
+        default=(project_path / "trained_models/bloom-1b4-sft").as_posix(),
+        type=str,
+    )
+    parser.add_argument('--device', default='auto', type=str)
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    if args.device == 'auto':
+        device = 'cuda' if torch.cuda.is_available() else 'cpu'
+    else:
+        device = args.device
+
+    # pretrained model
+    tokenizer = BloomTokenizerFast.from_pretrained(args.trained_model_path)
+    model = BloomForCausalLM.from_pretrained(args.trained_model_path)
+
+    model.eval()
+    model = model.to(device)
+    text = input('User：')
+    while True:
+        text = '<s>{}</s></s>'.format(text)
+        input_ids = tokenizer(text, return_tensors="pt").input_ids
+        input_ids = input_ids.to(device)
+        outputs = model.generate(input_ids, max_new_tokens=200, do_sample=True, top_p=0.85, temperature=0.35,
+                                 repetition_penalty=1.2, eos_token_id=tokenizer.eos_token_id)
+        rets = tokenizer.batch_decode(outputs)
+        output = rets[0].strip().replace(text, "").replace('</s>', "")
+        print("LLM：{}".format(output))
+        text = input('User：')
+
+
+if __name__ == '__main__':
+    main()

examples/exercises/firefly_bloom_1b4/ds_z3_config.json

@@ -0,0 +1,46 @@
+{
+  "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 200,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false,
+
+  "optimizer": {
+        "type": "Adam",
+        "params": {
+            "lr": "auto",
+            "betas": "auto",
+            "eps": "auto",
+            "weight_decay": "auto"
+        }
+    },
+  "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+  "zero_optimization": {
+    "stage": 3,
+    "overlap_comm": true,
+    "contiguous_gradients": true,
+    "sub_group_size": 1e9,
+    "reduce_bucket_size": "auto",
+    "stage3_prefetch_bucket_size": "auto",
+    "stage3_param_persistence_threshold": "auto",
+    "stage3_max_live_parameters": 1e9,
+    "stage3_max_reuse_distance": 1e9,
+    "stage3_gather_16bit_weights_on_model_save": true
+  },
+  "scheduler": {
+        "type": "WarmupLR",
+        "params": {
+            "warmup_min_lr": "auto",
+            "warmup_max_lr": "auto",
+            "warmup_num_steps": "auto"
+        }
+    }
+}

examples/exercises/firefly_bloom_1b4/run.sh

@@ -0,0 +1,192 @@
+#!/usr/bin/env bash
+
+# sh run.sh --stage -1 --stop_stage 2 --system_version centos --pretrained_model_name bloom-1b4-zh --final_model_name bloom-1b4-sft
+# sh run.sh --stage -1 --stop_stage 1 --system_version centos --pretrained_model_name bloom-1b4-zh
+# sh run.sh --stage 1 --stop_stage 1 --system_version centos --pretrained_model_name bloom-1b4-zh
+# sh run.sh --stage 2 --stop_stage 2 --system_version centos --pretrained_model_name bloom-1b4-zh --final_model_name bloom-1b4-sft
+
+# sh run.sh --stage 1 --stop_stage 1 --system_version windows --pretrained_model_name bloom-1b4-zh
+
+# params
+system_version="windows";
+verbose=true;
+stage=0 # start from 0 if you need to start from data preparation
+stop_stage=5
+pretrained_model_supplier=YeungNLP
+
+#pretrained_model_name=bloom-396m-zh
+#pretrained_model_name=bloom-820m-zh
+pretrained_model_name=bloom-1b4-zh
+
+final_model_name=final_model_name
+
+
+patience=0
+
+
+# parse options
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
+      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      old_value="(eval echo \\$$name)";
+      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval "${name}=\"$2\"";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+
+    *) break;
+  esac
+done
+
+
+$verbose && echo "system_version: ${system_version}"
+
+work_dir="$(pwd)"
+file_dir="${work_dir}/file_dir"
+cache_dir="${file_dir}/cache_dir"
+serialization_dir="${file_dir}/serialization_dir"
+
+data_dir="/data/tianxing/PycharmProjects/datasets/firefly_train_1_1m"
+pretrained_models_dir="${work_dir}/../../../pretrained_models/huggingface/${pretrained_model_supplier}"
+final_model_dir="${work_dir}/../../../trained_models/${final_model_name}";
+
+mkdir -p "${file_dir}"
+mkdir -p "${cache_dir}"
+mkdir -p "${serialization_dir}"
+mkdir -p "${data_dir}"
+mkdir -p "${pretrained_models_dir}"
+mkdir -p "${final_model_dir}"
+
+export PYTHONPATH="${work_dir}/../../.."
+
+if [ $system_version == "windows" ]; then
+  alias python3='C:/Users/tianx/PycharmProjects/virtualenv/Transformers/Scripts/python.exe'
+elif [ $system_version == "centos" ]; then
+  # conda activate Transformers
+  alias python3='/usr/local/miniconda3/envs/Transformers/bin/python3'
+elif [ $system_version == "ubuntu" ]; then
+  # conda activate Transformers
+  alias python3='/usr/local/miniconda3/envs/Transformers/bin/python3'
+fi
+
+
+function search_best_ckpt() {
+  patience="$1";
+
+  cd "${serialization_dir}" || exit 1
+  last_epoch=$(ls . | \
+               grep "checkpoint-*" | \
+               awk -F'[-]' '{print$2}' | \
+               sort -n | \
+               awk 'END {print}')
+
+  target_dir=
+  if [ -n "${last_epoch}" ]; then
+    target_epoch=$((last_epoch - patience))
+
+    for epoch_idx in $(ls . | grep "checkpoint-*" | awk -F'[-]' '{print$2}' | sort -nr):
+    do
+      if [ "${epoch_idx}" -le "${target_epoch}" ]; then
+        target_dir="checkpoint-${epoch_idx}";
+        break;
+      fi
+    done
+  fi
+
+  echo "${target_dir}"
+}
+
+
+if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
+  $verbose && echo "stage -1: download data"
+  cd "${data_dir}" || exit 1;
+
+  firefly_train_1_1m_size=$(/bin/ls -l firefly-train-1.1M.jsonl | awk '{print $5}')
+  if [ ! -e firefly-train-1.1M.jsonl ] || [ "${firefly_train_1_1m_size}" != "1171119212" ]; then
+    # rm firefly-train-1.1M.jsonl
+    wget -c https://huggingface.co/datasets/YeungNLP/firefly-train-1.1M/resolve/main/firefly-train-1.1M.jsonl
+  fi
+
+fi
+
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+  $verbose && echo "stage 0: download pretrained model"
+  cd "${work_dir}" || exit 1;
+  cd "${pretrained_models_dir}" || exit 1;
+
+  if [ ! -d "${pretrained_model_name}" ]; then
+    git clone "https://huggingface.co/${pretrained_model_supplier}/${pretrained_model_name}/"
+
+    cd "${pretrained_models_dir}/${pretrained_model_name}" || exit 1;
+    rm -rf .git
+    rm -rf flax_model.msgpack
+    rm -rf model.safetensors
+    rm -rf pytorch_model.bin
+    rm -rf tokenizer.json
+
+  fi
+
+  cd "${pretrained_models_dir}/${pretrained_model_name}" || exit 1;
+  if [ ! -e pytorch_model.bin ]; then
+    wget -c "https://huggingface.co/${pretrained_model_supplier}/${pretrained_model_name}/resolve/main/pytorch_model.bin"
+  fi
+
+  if [ ! -e tokenizer.json ]; then
+    wget -c "https://huggingface.co/${pretrained_model_supplier}/${pretrained_model_name}/resolve/main/tokenizer.json"
+  fi
+
+fi
+
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+  $verbose && echo "stage 1: train model"
+  cd "${work_dir}" || exit 1;
+  target_dir=$(search_best_ckpt "${patience}");
+
+  resume_from_checkpoint=
+  if [ -n "${target_dir}" ]; then
+  resume_from_checkpoint="${serialization_dir}/${target_dir}"
+    echo "resume_from_checkpoint: ${resume_from_checkpoint}"
+  fi
+
+  python3 1.train_model.py \
+  --train_file "${data_dir}/firefly-train-1.1M.jsonl" \
+  --pretrained_model_name_or_path "${pretrained_models_dir}/${pretrained_model_name}" \
+  --output_dir "${serialization_dir}" \
+  --cache_dir "${cache_dir}" \
+  --fp16 \
+  ${resume_from_checkpoint:+--resume_from_checkpoint $resume_from_checkpoint}
+
+fi
+
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+  $verbose && echo "stage 2: collect files"
+  target_dir=$(search_best_ckpt "${patience}");
+
+  cd "${work_dir}" || exit 1;
+
+  cp "${serialization_dir}/${target_dir}/pytorch_model.bin" "${final_model_dir}/pytorch_model.bin"
+
+  cp "${pretrained_models_dir}/${pretrained_model_name}/config.json" "${final_model_dir}/config.json"
+  cp "${pretrained_models_dir}/${pretrained_model_name}/special_tokens_map.json" "${final_model_dir}/special_tokens_map.json"
+  cp "${pretrained_models_dir}/${pretrained_model_name}/tokenizer_config.json" "${final_model_dir}/tokenizer_config.json"
+  cp "${pretrained_models_dir}/${pretrained_model_name}/tokenizer.json" "${final_model_dir}/tokenizer.json"
+
+fi

examples/exercises/firefly_bloom_1b4/stop.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+kill -9 `ps -aef | grep 'Transformers/bin/python3' | grep -v grep | awk '{print $2}' | sed 's/\n/ /'`
+kill -9 `ps -aef | grep 'run.sh' | grep -v grep | awk '{print $2}' | sed 's/\n/ /'`

main.py

@@ -0,0 +1,121 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+
+import gradio as gr
+import torch
+from transformers import BloomTokenizerFast, BloomForCausalLM
+
+from project_settings import project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--trained_model_path',
+        default=(project_path / "trained_models/bloom-1b4-sft").as_posix(),
+        type=str,
+    )
+    parser.add_argument('--device', default='auto', type=str)
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    if args.device == 'auto':
+        device = 'cuda' if torch.cuda.is_available() else 'cpu'
+    else:
+        device = args.device
+
+    # pretrained model
+    tokenizer = BloomTokenizerFast.from_pretrained(args.trained_model_path)
+    model = BloomForCausalLM.from_pretrained(args.trained_model_path)
+
+    description = """
+    FireflyBloom1b4
+    
+    基于 [YeungNLP/bloom-1b4-zh](https://huggingface.co/YeungNLP/bloom-1b4-zh) 预训练模型,
+    基于 [YeungNLP/firefly-train-1.1M](https://huggingface.co/datasets/YeungNLP/firefly-train-1.1M) 数据集, 
+    训练的等同于 [YeungNLP/firefly-bloom-1b4](https://huggingface.co/YeungNLP/firefly-bloom-1b4) 的问答模型. 
+    
+    训练代码是自己编写的, 在 examples 里, 总共训练了 3 个 epoch. 感觉效果还可以. 
+
+    """
+
+    def fn(text: str,
+           max_new_tokens: int = 200,
+           top_p: float = 0.85,
+           temperature: float = 0.35,
+           repetition_penalty: float = 1.2
+           ):
+        print(text)
+        text = '<s>{}</s></s>'.format(text)
+        input_ids = tokenizer(text, return_tensors="pt").input_ids
+        input_ids = input_ids.to(device)
+        outputs = model.generate(input_ids,
+                                 max_new_tokens=max_new_tokens,
+                                 do_sample=True,
+                                 top_p=top_p,
+                                 temperature=temperature,
+                                 repetition_penalty=repetition_penalty,
+                                 eos_token_id=tokenizer.eos_token_id
+                                 )
+        rets = tokenizer.batch_decode(outputs)
+        output = rets[0].strip().replace(text, "").replace('</s>', "")
+        print(output)
+        return output
+
+    demo = gr.Interface(
+        fn=fn,
+        inputs=[
+            gr.Text(label="text"),
+            gr.Number(value=200, label="max_new_tokens"),
+            gr.Slider(minimum=0, maximum=1, value=0.85, label="top_p"),
+            gr.Slider(minimum=0, maximum=1, value=0.35, label="temperature"),
+            gr.Number(value=1.2, label="repetition_penalty"),
+        ],
+        outputs=[gr.Text(label="output")],
+        examples=[
+            [
+                "将下面句子翻译成现代文：\n石中央又生一树，高百余尺，条干偃阴为五色，翠叶如盘，花径尺余，色深碧，蕊深红，异香成烟，著物霏霏。",
+                200, 0.85, 0.35, 1.2
+            ],
+            [
+                "实体识别: 1949年10月1日，人们在北京天安门广场参加开国大典。",
+                200, 0.85, 0.35, 1.2
+            ],
+            [
+                "把这句话翻译成英文: 1949年10月1日，人们在北京天安门广场参加开国大典。",
+                200, 0.85, 0.35, 1.2
+            ],
+            [
+                "晚上睡不着该怎么办. 请给点详细的介绍.",
+                200, 0.85, 0.35, 1.2
+            ],
+            [
+                "将下面的句子翻译成文言文：结婚率下降, 离婚率暴增, 生育率下降, 人民焦虑迷茫, 到底是谁的错.",
+                200, 0.85, 0.35, 1.2
+            ],
+            [
+                "对联：厌烟沿檐烟燕眼.",
+                200, 0.85, 0.35, 1.2
+            ],
+            [
+                "写一首咏雪的古诗, 标题为 \"沁园春, 雪\".",
+                200, 0.85, 0.35, 1.2
+            ],
+        ],
+        examples_per_page=50,
+        title="Firefly Bloom 1b4",
+        description=description,
+    )
+    demo.launch()
+
+    return
+
+
+if __name__ == '__main__':
+    main()

project_settings.py

@@ -0,0 +1,20 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import os
+from pathlib import Path
+
+from toolbox.os.environment import EnvironmentManager
+
+
+project_path = os.path.abspath(os.path.dirname(__file__))
+project_path = Path(project_path)
+
+
+environment = EnvironmentManager(
+    path=os.path.join(project_path, 'dotenv'),
+    env=os.environ.get('environment', 'dev'),
+)
+
+
+if __name__ == '__main__':
+    pass

requirements.txt

@@ -0,0 +1,11 @@
+gradio==3.20.1
+pydantic==1.10.12
+thinc==7.4.6
+spacy==2.3.9
+transformers==4.30.2
+numpy==1.21.4
+pandas==1.2.5
+tqdm==4.62.3
+torch==1.13.0
+datasets
+python-dotenv==1.0.0

toolbox/json/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/json/misc.py

@@ -0,0 +1,63 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+from typing import Callable
+
+
+def traverse(js, callback: Callable, *args, **kwargs):
+    if isinstance(js, list):
+        result = list()
+        for l in js:
+            l = traverse(l, callback, *args, **kwargs)
+            result.append(l)
+        return result
+    elif isinstance(js, tuple):
+        result = list()
+        for l in js:
+            l = traverse(l, callback, *args, **kwargs)
+            result.append(l)
+        return tuple(result)
+    elif isinstance(js, dict):
+        result = dict()
+        for k, v in js.items():
+            k = traverse(k, callback, *args, **kwargs)
+            v = traverse(v, callback, *args, **kwargs)
+            result[k] = v
+        return result
+    elif isinstance(js, int):
+        return callback(js, *args, **kwargs)
+    elif isinstance(js, str):
+        return callback(js, *args, **kwargs)
+    else:
+        return js
+
+
+def demo1():
+    d = {
+        "env": "ppe",
+        "mysql_connect": {
+            "host": "$mysql_connect_host",
+            "port": 3306,
+            "user": "callbot",
+            "password": "NxcloudAI2021!",
+            "database": "callbot_ppe",
+            "charset": "utf8"
+        },
+        "es_connect": {
+            "hosts": ["10.20.251.8"],
+            "http_auth": ["elastic", "ElasticAI2021!"],
+            "port": 9200
+        }
+    }
+
+    def callback(s):
+        if isinstance(s, str) and s.startswith('$'):
+            return s[1:]
+        return s
+
+    result = traverse(d, callback=callback)
+    print(result)
+    return
+
+
+if __name__ == '__main__':
+    demo1()

toolbox/os/__init__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+
+
+if __name__ == '__main__':
+    pass

toolbox/os/environment.py

@@ -0,0 +1,114 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import json
+import os
+
+from dotenv import load_dotenv
+from dotenv.main import DotEnv
+
+from toolbox.json.misc import traverse
+
+
+class EnvironmentManager(object):
+    def __init__(self, path, env, override=False):
+        filename = os.path.join(path, '{}.env'.format(env))
+        self.filename = filename
+
+        load_dotenv(
+            dotenv_path=filename,
+            override=override
+        )
+
+        self._environ = dict()
+
+    def open_dotenv(self, filename: str = None):
+        filename = filename or self.filename
+        dotenv = DotEnv(
+            dotenv_path=filename,
+            stream=None,
+            verbose=False,
+            interpolate=False,
+            override=False,
+            encoding="utf-8",
+        )
+        result = dotenv.dict()
+        return result
+
+    def get(self, key, default=None, dtype=str):
+        result = os.environ.get(key)
+        if result is None:
+            if default is None:
+                result = None
+            else:
+                result = default
+        else:
+            result = dtype(result)
+        self._environ[key] = result
+        return result
+
+
+_DEFAULT_DTYPE_MAP = {
+    'int': int,
+    'float': float,
+    'str': str,
+    'json.loads': json.loads
+}
+
+
+class JsonConfig(object):
+    """
+    将 json 中, 形如 `$float:threshold` 的值, 处理为:
+    从环境变量中查到 threshold, 再将其转换为 float 类型.
+    """
+    def __init__(self, dtype_map: dict = None, environment: EnvironmentManager = None):
+        self.dtype_map = dtype_map or _DEFAULT_DTYPE_MAP
+        self.environment = environment or os.environ
+
+    def sanitize_by_filename(self, filename: str):
+        with open(filename, 'r', encoding='utf-8') as f:
+            js = json.load(f)
+
+        return self.sanitize_by_json(js)
+
+    def sanitize_by_json(self, js):
+        js = traverse(
+            js,
+            callback=self.sanitize,
+            environment=self.environment
+        )
+        return js
+
+    def sanitize(self, string, environment):
+        """支持 $ 符开始的, 环境变量配置"""
+        if isinstance(string, str) and string.startswith('$'):
+            dtype, key = string[1:].split(':')
+            dtype = self.dtype_map[dtype]
+
+            value = environment.get(key)
+            if value is None:
+                raise AssertionError('environment not exist. key: {}'.format(key))
+
+            value = dtype(value)
+            result = value
+        else:
+            result = string
+        return result
+
+
+def demo1():
+    import json
+
+    from project_settings import project_path
+
+    environment = EnvironmentManager(
+        path=os.path.join(project_path, 'server/callbot_server/dotenv'),
+        env='dev',
+    )
+    init_scenes = environment.get(key='init_scenes', dtype=json.loads)
+    print(init_scenes)
+    print(environment._environ)
+    return
+
+
+if __name__ == '__main__':
+    demo1()

toolbox/os/other.py

@@ -0,0 +1,9 @@
+import os
+import inspect
+
+
+def pwd():
+    """你在哪个文件调用此函数, 它就会返回那个文件所在的 dir 目标"""
+    frame = inspect.stack()[1]
+    module = inspect.getmodule(frame[0])
+    return os.path.dirname(os.path.abspath(module.__file__))

trained_models/bloom-1b4-sft/config.json

@@ -0,0 +1,33 @@
+{
+  "_name_or_path": "YeungNLP/bloom-1b4-zh",
+  "apply_residual_connection_post_layernorm": false,
+  "architectures": [
+    "BloomForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attention_softmax_in_fp32": true,
+  "bias_dropout_fusion": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_dropout": 0.0,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "masked_softmax_fusion": true,
+  "model_type": "bloom",
+  "n_embed": 2048,
+  "n_inner": null,
+  "n_layer": 24,
+  "num_attention_heads": 16,
+  "offset_alibi": 100,
+  "pad_token_id": 3,
+  "pretraining_tp": 2,
+  "seq_length": 4096,
+  "skip_bias_add": true,
+  "skip_bias_add_qkv": false,
+  "slow_but_exact": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.20.0",
+  "unk_token_id": 0,
+  "use_cache": true,
+  "vocab_size": 46145
+}

trained_models/bloom-1b4-sft/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b3d273ca775ee59871273c92c84803369423f045987057c3152b85f92244a6d
+size 5212546569

trained_models/bloom-1b4-sft/special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

trained_models/bloom-1b4-sft/tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "name_or_path": "/Users/jianxin.yang/Desktop/pretrain_models/bloom-6b4-zh",
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "special_tokens_map_file": null,
+  "tokenizer_class": "BloomTokenizer",
+  "unk_token": "<unk>"
+}