[update]add model

examples/exercises/chinese_porn_novel/1.prepare_data.py

@@ -0,0 +1,75 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import json
+import os
+import random
+import sys
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, '../../../'))
+
+from datasets import Dataset, DatasetDict, IterableDataset, load_dataset
+from tqdm import tqdm
+
+from project_settings import project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--dataset_path", default="qgyd2021/h_novel", type=str)
+    # parser.add_argument("--dataset_name", default="ltxsba_500m", type=str)
+    parser.add_argument("--dataset_name", default="ltxsba_5gb", type=str)
+    parser.add_argument("--dataset_split", default="train", type=str)
+    parser.add_argument(
+        "--dataset_cache_dir",
+        default=(project_path / "hub_datasets").as_posix(),
+        type=str
+    )
+    parser.add_argument("--train_subset", default="train.jsonl", type=str)
+    parser.add_argument("--valid_subset", default="valid.jsonl", type=str)
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    dataset_dict = load_dataset(
+        path=args.dataset_path,
+        name=args.dataset_name,
+        # split=args.dataset_split,
+        cache_dir=args.dataset_cache_dir,
+        streaming=True,
+    )
+
+    train_dataset = dataset_dict["train"]
+
+    with open(args.train_subset, "w", encoding="utf-8") as ftrain, \
+        open(args.valid_subset, "w", encoding="utf-8") as fvalid:
+        for sample in tqdm(train_dataset):
+            # print(sample)
+
+            source = sample["source"]
+            idx = sample["idx"]
+            filename = sample["filename"]
+            novel_name = sample["novel_name"]
+            row_idx = sample["row_idx"]
+            text = sample["text"]
+
+            row = {
+                "text": text
+            }
+            row = json.dumps(row, ensure_ascii=False)
+
+            if random.random() < 0.95:
+                ftrain.write("{}\n".format(row))
+            else:
+                fvalid.write("{}\n".format(row))
+
+    return
+
+
+if __name__ == '__main__':
+    main()

examples/exercises/chinese_porn_novel/2.train_model.py

@@ -0,0 +1,203 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+"""
+参考链接:
+https://www.thepythoncode.com/article/pretraining-bert-huggingface-transformers-in-python
+https://github.com/huggingface/transformers/blob/main/examples/pytorch/language-modeling/run_clm.py
+
+"""
+import argparse
+from itertools import chain
+import os
+from pathlib import Path
+import platform
+
+from datasets import Dataset, DatasetDict, IterableDataset, load_dataset
+import torch
+from transformers.data.data_collator import DataCollatorForLanguageModeling
+from transformers.models.gpt2.modeling_gpt2 import GPT2LMHeadModel
+from transformers.models.bert.tokenization_bert import BertTokenizer
+from transformers.trainer import Trainer
+from transformers.training_args import TrainingArguments
+
+from project_settings import project_path
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--pretrained_model_name_or_path",
+        default=(project_path / "pretrained_models/gpt2-chinese-cluecorpussmall").as_posix(),
+        type=str
+    )
+
+    parser.add_argument("--train_subset", default="train.jsonl", type=str)
+    parser.add_argument("--valid_subset", default="valid.jsonl", type=str)
+
+    parser.add_argument("--output_dir", default="serialization_dir", type=str)
+    parser.add_argument("--overwrite_output_dir", action="store_true")
+    parser.add_argument("--evaluation_strategy", default="no", choices=["no", "steps", "epoch"], type=str)
+    parser.add_argument("--per_device_train_batch_size", default=8, type=int)
+    parser.add_argument("--gradient_accumulation_steps", default=4, type=int)
+    parser.add_argument("--learning_rate", default=1e-5, type=float)
+    parser.add_argument("--weight_decay", default=0, type=float)
+    parser.add_argument("--max_grad_norm", default=1.0, type=float)
+    parser.add_argument("--num_train_epochs", default=3.0, type=float)
+    parser.add_argument("--max_steps", default=-1, type=int)
+    parser.add_argument("--lr_scheduler_type", default="cosine", type=str)
+    parser.add_argument("--warmup_ratio", default=0.0, type=float)
+    parser.add_argument("--warmup_steps", default=3000, type=int)
+    parser.add_argument("--logging_steps", default=300, type=int)
+    parser.add_argument("--save_strategy", default="steps", type=str)
+    parser.add_argument("--save_steps", default=500, type=int)
+    parser.add_argument("--save_total_limit", default=3, type=int)
+    parser.add_argument("--no_cuda", action="store_true")
+    parser.add_argument("--seed", default=3407, type=str, help="https://arxiv.org/abs/2109.08203")
+    # parser.add_argument("--fp16", action="store_true")
+    parser.add_argument("--fp16", action="store_false")
+    parser.add_argument("--half_precision_backend", default="auto", type=str)
+    parser.add_argument("--dataloader_num_workers", default=5, type=int)
+    parser.add_argument("--disable_tqdm", action="store_false")
+    parser.add_argument("--remove_unused_columns", action="store_false")
+    # parser.add_argument("--deepspeed", default="ds_z3_config.json", type=str)
+    parser.add_argument("--deepspeed", default=None, type=str)
+    parser.add_argument("--optim", default="adamw_hf", type=str)
+    parser.add_argument("--report_to", default="tensorboard", type=str)
+    parser.add_argument("--resume_from_checkpoint", default=None, type=str)
+    # parser.add_argument("--gradient_checkpointing", action="store_true")
+    parser.add_argument("--gradient_checkpointing", action="store_false")
+
+    parser.add_argument("--truncate_longer_samples", action="store_true")
+    # parser.add_argument("--truncate_longer_samples", action="store_false")
+    parser.add_argument("--max_seq_length", default=1024, type=int)
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    # dataset
+    dataset_dict = DatasetDict()
+    train_data_files = [args.train_subset]
+    dataset_dict["train"] = load_dataset(
+        path="json", data_files=[str(file) for file in train_data_files]
+    )["train"]
+    valid_data_files = [args.valid_subset]
+    dataset_dict["valid"] = load_dataset(
+        path="json", data_files=[str(file) for file in valid_data_files]
+    )["train"]
+
+    print(dataset_dict)
+
+    # model
+    tokenizer = BertTokenizer.from_pretrained(args.pretrained_model_name_or_path)
+    model = GPT2LMHeadModel.from_pretrained(args.pretrained_model_name_or_path)
+
+    def encode_with_truncation(examples):
+        outputs = tokenizer.__call__(examples['text'],
+                                     truncation=True,
+                                     padding='max_length',
+                                     max_length=args.max_seq_length,
+                                     return_special_tokens_mask=True)
+        return outputs
+
+    def encode_without_truncation(examples):
+        outputs = tokenizer.__call__(examples['text'],
+                                     return_special_tokens_mask=True)
+        return outputs
+
+    def group_texts(examples):
+        concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()}
+        total_length = len(concatenated_examples[list(examples.keys())[0]])
+        if total_length >= args.max_seq_length:
+            total_length = (total_length // args.max_seq_length) * args.max_seq_length
+
+        result = {
+            k: [t[i: i + args.max_seq_length] for i in range(0, total_length, args.max_seq_length)]
+            for k, t in concatenated_examples.items()
+        }
+        return result
+
+    if args.truncate_longer_samples:
+        dataset_dict = dataset_dict.map(
+            encode_with_truncation,
+            batched=True,
+            drop_last_batch=True,
+            keep_in_memory=False,
+            # num_proc=None if platform.system() == 'Windows' else os.cpu_count() // 2,
+            num_proc=None,
+        )
+        dataset_dict.set_format(type="torch", columns=["input_ids", "attention_mask"])
+    else:
+        dataset_dict = dataset_dict.map(
+            encode_without_truncation,
+            batched=True,
+            drop_last_batch=True,
+            keep_in_memory=False,
+            # num_proc=None if platform.system() == 'Windows' else os.cpu_count() // 2,
+            num_proc=None,
+        )
+        dataset_dict.set_format(type="torch", columns=["input_ids", "attention_mask"])
+
+        dataset_dict = dataset_dict.map(
+            group_texts,
+            batched=True,
+            drop_last_batch=True,
+            keep_in_memory=False,
+            # num_proc=None if platform.system() == 'Windows' else os.cpu_count() // 2,
+            num_proc=None,
+        )
+        dataset_dict.set_format("torch")
+
+    data_collator = DataCollatorForLanguageModeling(
+        tokenizer=tokenizer, mlm=False
+    )
+
+    training_args = TrainingArguments(
+        output_dir=args.output_dir,
+        overwrite_output_dir=args.overwrite_output_dir,
+        evaluation_strategy=args.evaluation_strategy,
+        per_device_train_batch_size=args.per_device_train_batch_size,
+        gradient_accumulation_steps=args.gradient_accumulation_steps,
+        learning_rate=args.learning_rate,
+        num_train_epochs=args.num_train_epochs,
+        max_steps=args.max_steps,
+        lr_scheduler_type=args.lr_scheduler_type,
+        warmup_steps=args.warmup_steps,
+        logging_steps=args.logging_steps,
+        save_steps=args.save_steps,
+        save_total_limit=args.save_total_limit,
+        no_cuda=args.no_cuda,
+        fp16=args.fp16,
+        half_precision_backend=args.half_precision_backend,
+        # deepspeed=args.deepspeed,
+        report_to=args.report_to,
+        resume_from_checkpoint=args.resume_from_checkpoint,
+        gradient_checkpointing=args.gradient_checkpointing,
+    )
+
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        data_collator=data_collator,
+        train_dataset=dataset_dict["train"],
+    )
+    train_result = trainer.train()
+
+    # 保存最好的 checkpoint
+    final_save_path = os.path.join(training_args.output_dir, "final")
+    trainer.save_model(final_save_path)  # Saves the tokenizer too
+    # 保存训练指标
+    metrics = train_result.metrics
+    trainer.log_metrics("train", metrics)
+    trainer.save_metrics("train", metrics)
+    trainer.save_state()
+
+    tokenizer.save_pretrained(final_save_path)
+    return
+
+
+if __name__ == '__main__':
+    main()

examples/exercises/chinese_porn_novel/3.test_model.py

@@ -0,0 +1,88 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import os
+import sys
+
+pwd = os.path.abspath(os.path.dirname(__file__))
+sys.path.append(os.path.join(pwd, '../../../'))
+
+import torch
+from transformers.models.gpt2.modeling_gpt2 import GPT2LMHeadModel
+from transformers.models.bert.tokenization_bert import BertTokenizer
+
+from project_settings import project_path
+
+
+def get_args():
+    """
+python3 3.test_model.py \
+--repetition_penalty 1.2 \
+--trained_model_path /data/tianxing/PycharmProjects/Transformers/trained_models/gpt2_chinese_h_novel
+
+python3 3.test_model.py \
+--trained_model_path /data/tianxing/PycharmProjects/Transformers/trained_models/gpt2_chinese_h_novel
+
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '--trained_model_path',
+        default=(project_path / "pretrained_models/gpt2-chinese-cluecorpussmall").as_posix(),
+        type=str,
+    )
+    parser.add_argument('--device', default='auto', type=str)
+
+    parser.add_argument('--max_new_tokens', default=512, type=int)
+    parser.add_argument('--top_p', default=0.85, type=float)
+    parser.add_argument('--temperature', default=0.35, type=float)
+    parser.add_argument('--repetition_penalty', default=1.2, type=float)
+
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    if args.device == 'auto':
+        device = 'cuda' if torch.cuda.is_available() else 'cpu'
+    else:
+        device = args.device
+
+    # pretrained model
+    tokenizer = BertTokenizer.from_pretrained(args.trained_model_path)
+    model = GPT2LMHeadModel.from_pretrained(args.trained_model_path)
+
+    model.eval()
+    model = model.to(device)
+
+    while True:
+        text = input('prefix: ')
+
+        if text == "Quit":
+            break
+        text = '{}'.format(text)
+        input_ids = tokenizer(text, return_tensors="pt").input_ids
+        input_ids = input_ids[:, :-1]
+        # print(input_ids)
+        # print(type(input_ids))
+        input_ids = input_ids.to(device)
+
+        outputs = model.generate(input_ids,
+                                 max_new_tokens=512,
+                                 do_sample=True,
+                                 top_p=args.top_p,
+                                 temperature=args.temperature,
+                                 repetition_penalty=args.repetition_penalty,
+                                 eos_token_id=tokenizer.sep_token_id,
+                                 pad_token_id=tokenizer.pad_token_id
+                                 )
+        rets = tokenizer.batch_decode(outputs)
+        output = rets[0].replace(" ", "").replace("[CLS]", "").replace("[SEP]", "")
+        print("{}".format(output))
+
+    return
+
+
+if __name__ == '__main__':
+    main()

examples/exercises/chinese_porn_novel/README.md

@@ -0,0 +1,8 @@
+## 预训练 GPT 模型
+
+```text
+参考链接: 
+https://huggingface.co/docs/transformers/model_doc/openai-gpt
+https://huggingface.co/learn/nlp-course/chapter7/6
+
+```

examples/exercises/chinese_porn_novel/run.sh

@@ -0,0 +1,146 @@
+#!/usr/bin/env bash
+
+# nohup sh run.sh --stage 0 --stop_stage 1 --system_version centos &
+# sh run.sh --stage 0 --stop_stage 1 --system_version windows
+# sh run.sh --stage 0 --stop_stage 0 --system_version centos
+# sh run.sh --stage 2 --stop_stage 2 --system_version centos --checkpoint_name final
+# sh run.sh --stage -1 --stop_stage 1
+
+# bitsandbytes
+export LD_LIBRARY_PATH="/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+
+# params
+system_version="windows";
+verbose=true;
+stage=0 # start from 0 if you need to start from data preparation
+stop_stage=5
+
+pretrained_model_name=gpt2-chinese-cluecorpussmall
+
+train_subset=train.jsonl
+valid_subset=valid.jsonl
+
+final_model_name=gpt2_chinese_h_novel
+
+checkpoint_name=final
+
+# parse options
+while true; do
+  [ -z "${1:-}" ] && break;  # break if there are no arguments
+  case "$1" in
+    --*) name=$(echo "$1" | sed s/^--// | sed s/-/_/g);
+      eval '[ -z "${'"$name"'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
+      old_value="(eval echo \\$$name)";
+      if [ "${old_value}" == "true" ] || [ "${old_value}" == "false" ]; then
+        was_bool=true;
+      else
+        was_bool=false;
+      fi
+
+      # Set the variable to the right value-- the escaped quotes make it work if
+      # the option had spaces, like --cmd "queue.pl -sync y"
+      eval "${name}=\"$2\"";
+
+      # Check that Boolean-valued arguments are really Boolean.
+      if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
+        echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
+        exit 1;
+      fi
+      shift 2;
+      ;;
+
+    *) break;
+  esac
+done
+
+$verbose && echo "system_version: ${system_version}"
+
+work_dir="$(pwd)"
+file_dir="$(pwd)/file_dir"
+pretrained_models_dir="${work_dir}/../../../pretrained_models";
+serialization_dir="${file_dir}/serialization_dir"
+final_model_dir="${work_dir}/../../../trained_models/${final_model_name}";
+
+mkdir -p "${file_dir}"
+mkdir -p "${pretrained_models_dir}"
+mkdir -p "${serialization_dir}"
+mkdir -p "${final_model_dir}"
+
+
+export PYTHONPATH="${work_dir}/../../.."
+
+
+if [ $system_version == "windows" ]; then
+  alias python3='C:/Users/tianx/PycharmProjects/virtualenv/Transformers/Scripts/python.exe'
+elif [ $system_version == "centos" ]; then
+  # conda activate Transformers
+  alias python3='/usr/local/miniconda3/envs/Transformers/bin/python3'
+elif [ $system_version == "ubuntu" ]; then
+  # conda activate Transformers
+  alias python3='/usr/local/miniconda3/envs/Transformers/bin/python3'
+fi
+
+
+declare -A pretrained_model_dict
+pretrained_model_dict=(
+  ["gpt2-chinese-cluecorpussmall"]="https://huggingface.co/uer/gpt2-chinese-cluecorpussmall"
+  ["gpt2"]="https://huggingface.co/gpt2"
+  ["japanese-gpt2-medium"]="https://huggingface.co/rinna/japanese-gpt2-medium"
+
+)
+pretrained_model_dir="${pretrained_models_dir}/${pretrained_model_name}"
+
+
+if [ ${stage} -le -1 ] && [ ${stop_stage} -ge -1 ]; then
+  $verbose && echo "stage -1: download pretrained model"
+  cd "${file_dir}" || exit 1;
+
+  if [ ! -d "${pretrained_model_dir}" ]; then
+    cd "${pretrained_models_dir}" || exit 1;
+
+    repository_url="${pretrained_model_dict[${pretrained_model_name}]}"
+    git clone "${repository_url}"
+
+    cd "${pretrained_model_dir}" || exit 1;
+    rm flax_model.msgpack && rm pytorch_model.bin && rm tf_model.h5
+    wget "${repository_url}/resolve/main/pytorch_model.bin"
+  fi
+fi
+
+
+if [ ${stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
+  $verbose && echo "stage 0: prepare data"
+  cd "${work_dir}" || exit 1;
+
+  python3 1.prepare_data.py \
+  --train_subset "${file_dir}/${train_subset}" \
+  --valid_subset "${file_dir}/${valid_subset}" \
+
+fi
+
+
+if [ ${stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
+  $verbose && echo "stage 1: train model"
+  cd "${work_dir}" || exit 1;
+
+  python3 2.train_model.py \
+  --train_subset "${file_dir}/${train_subset}" \
+  --valid_subset "${file_dir}/${valid_subset}" \
+  --pretrained_model_name_or_path "${pretrained_models_dir}/${pretrained_model_name}" \
+  --output_dir "${serialization_dir}"
+
+fi
+
+
+if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
+  $verbose && echo "stage 2: collect files"
+  cd "${work_dir}" || exit 1;
+
+  cp "${serialization_dir}/${checkpoint_name}/pytorch_model.bin" "${final_model_dir}/pytorch_model.bin"
+
+  cp "${pretrained_models_dir}/${pretrained_model_name}/config.json" "${final_model_dir}/config.json"
+  cp "${pretrained_models_dir}/${pretrained_model_name}/special_tokens_map.json" "${final_model_dir}/special_tokens_map.json"
+  cp "${pretrained_models_dir}/${pretrained_model_name}/tokenizer_config.json" "${final_model_dir}/tokenizer_config.json"
+  cp "${pretrained_models_dir}/${pretrained_model_name}/vocab.txt" "${final_model_dir}/vocab.txt"
+
+fi

examples/exercises/chinese_porn_novel/stop.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+kill -9 `ps -aef | grep 'run.sh' | grep -v grep | awk '{print $2}' | sed 's/\n/ /'`
+
+kill -9 `ps -aef | grep 'Transformers/bin/python3' | grep -v grep | awk '{print $2}' | sed 's/\n/ /'`

main.py

@@ -117,7 +117,9 @@ def main():
             yield output
 
     model_name_choices = ["trained_models/lib_service_4chan"] \
-        if platform.system() == "Windows" else ["qgyd2021/lib_service_4chan", "qgyd2021/chinese_chitchat"]
+        if platform.system() == "Windows" else \
+        ["qgyd2021/lib_service_4chan", "qgyd2021/chinese_chitchat", "qgyd2021/chinese_porn_novel"]
+
     demo = gr.Interface(
         fn=fn_stream,
         inputs=[
@@ -133,6 +135,8 @@ def main():
         examples=[
             ["怎样擦屁股才能擦的干净", 512, 0.75, 0.35, 1.2, "qgyd2021/lib_service_4chan", True],
             ["你好", 512, 0.75, 0.35, 1.2, "qgyd2021/chinese_chitchat", True],
+            ["白洁走到床边并脱去内衣, 一双硕大的", 512, 0.75, 0.35, 1.2, "qgyd2021/chinese_porn_novel", False],
+            ["男人走进房间, 上床, 压上", 512, 0.75, 0.35, 1.2, "qgyd2021/chinese_porn_novel", False],
         ],
         cache_examples=False,
         examples_per_page=50,