add gpt data generation and analysis

build_openprompt.py

@@ -3,6 +3,9 @@ import pandas as pd
 import json
 import random
 
+from torch.nn.utils.rnn import pad_sequence
+
+
 from tqdm import tqdm
 
 
@@ -12,31 +15,34 @@ samples = {
         }
 little = False
 all_loaded_sample = 400000
-
+normal = True # 全部读取，非采样方式
 s_pro = all_loaded_sample / 1e+7
 # 读取概率
-with open("./data/prompts.csv") as f:
+with open("./data/cleaned_oie_prompts.csv") as f:
     csv_reader = csv.DictReader(f)
     process_reader = tqdm(enumerate(csv_reader))
     for row_number, row in process_reader:
         num_samples = len(samples['x'])
         process_reader.set_description(f"got data num: {num_samples}")
-        if random.uniform(0, 1) > s_pro:
-            continue
+        if not normal:
+            if random.uniform(0, 1) > s_pro:
+                continue
+            if len(samples["x"]) > all_loaded_sample:
+                break
+        else:
+            if row['prompt'] == "":
+                continue
         if little:
             if len(samples["x"]) > 100:
                 break
-        if len(samples["x"]) > all_loaded_sample:
-            break
         
         datum = row
-        prompt = datum['prompt']
-        modifiers = json.loads(datum['raw_data'])['modifiers']
-        if len(modifiers) < 4:
-            continue
-        
-        # TODO: 外挂一个entity识别，过滤掉存在entity实体的数据
-        
+        # prompt = datum['prompt']
+        prompt = ",".join(eval(datum['raw_data'])['modifiers'])
+        if not normal:
+            modifiers = eval(datum['raw_data'])['modifiers']
+            if len(modifiers) < 4:
+                continue        
         label = prompt
         x = prompt
         # 小文本到大文本，因此x更小，同时x按照6:3:1的比例分配

corenlp_openie.py

@@ -0,0 +1,104 @@
+import os
+import re
+import csv
+import json
+import jsonlines
+from tqdm import tqdm
+from stanfordcorenlp import StanfordCoreNLP
+
+import concurrent.futures
+
+
+nlp = StanfordCoreNLP('./stanford-corenlp-4.5.5')
+
+SOURCE_FILE = "./data/raw_oie_source.jsonl"
+
+def oie_extract(sentence):
+    output = nlp.annotate(sentence, properties={
+        'annotators': 'tokenize, ssplit, pos, depparse, parse, openie',
+        'outputFormat': 'json'
+        })
+    try:
+        data = json.loads(output)
+        sentences_ie = [i['openie'] for i in data['sentences'] if len(i['openie']) > 0]
+        oie_result = [max([sub["object"] for sub in sen], key=len) for sen in sentences_ie]
+        central_result = [sen[0]["subject"] for sen in sentences_ie][1:]
+
+        result = central_result + oie_result
+        result = ",".join(result)
+    except Exception as e:
+        print(f"An error occurred output: {output}")
+        result = ""
+    return result
+
+def process_sentence(sentence):
+    row_data = {'raw_data': {'modifiers': sentence.split(".")}, 'prompt': ''}
+    oie_prompt = oie_extract(sentence)
+    row_data['prompt'] = oie_prompt
+    return row_data
+
+def get_sentences(path):
+    if not os.path.exists(SOURCE_FILE):
+        raise FileNotFoundError(f"{SOURCE_FILE} not found.")
+    
+    with jsonlines.open(path) as reader:
+        for obj in reader:
+            yield obj['description']
+
+def main():
+    file_name = "./data/oie_prompts.csv"
+    fieldnames = ['prompt', 'raw_data']
+    csvfile = open(file_name, 'w', newline='', encoding='utf-8')
+    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+    writer.writeheader()
+    
+    # for sentence in tqdm(get_sentences(SOURCE_FILE), desc="extracting oie prompts"):
+    #     row_data = {'raw_data': {'modifiers': sentence.split(".")}, "prompt": ""}
+    #     oie_prompt = oie_extract(sentence)
+    #     row_data['prompt'] = oie_prompt
+    #     writer.writerow(row_data)
+    
+    with concurrent.futures.ThreadPoolExecutor() as executor:
+        results = list(tqdm(executor.map(process_sentence, get_sentences(SOURCE_FILE)), 
+                           total=len(list(get_sentences(SOURCE_FILE))), 
+                           desc="extracting oie prompts"))
+    
+    for result in results:
+        writer.writerow(result)
+    
+def remove_chinese(text):
+    pattern = re.compile(r'[\u4e00-\u9fa5]')
+    result = re.sub(pattern, '', text)
+    return result
+    
+    
+def remove_special_chars(text):
+    pattern = re.compile(r'[^\w\s.,]')
+    result = re.sub(pattern, '', text)
+    return result
+
+def cleaning_dataset():
+    """只清理oie_prompts.csv，保存在cleaned_oie_prompts.csv中"""
+    file_name = "./data/cleaned_oie_prompts.csv"
+    fieldnames = ['prompt', 'raw_data']
+    csvfile = open(file_name, 'w', newline='', encoding='utf-8')
+    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+    writer.writeheader()
+    with open("./data/oie_prompts.csv") as f:
+        csv_reader = csv.DictReader(f)
+        process_reader = tqdm(enumerate(csv_reader))
+        for row_number, row in process_reader:
+            datum = row
+            
+            cleaned_prompts = remove_special_chars(remove_chinese(datum['prompt']))
+            joined_modifiers = ",".join(eval(datum['raw_data'])['modifiers'])
+            cleaned_modifiers = remove_special_chars(remove_chinese(joined_modifiers))
+            row_data = {'raw_data': {'modifiers': cleaned_modifiers.split(",")}, "prompt": cleaned_prompts}
+            writer.writerow(row_data)
+
+
+if __name__ == '__main__':
+    # main()
+    cleaning_dataset()
+    
+    

generation_test.py

@@ -0,0 +1,101 @@
+import os
+import spacy
+from accelerate import PartialState
+from accelerate.utils import set_seed
+
+from gpt2_generation import Translator
+from gpt2_generation import generate_prompt, MODEL_CLASSES
+
+os.environ["http_proxy"] = "http://127.0.0.1:7890"
+os.environ["https_proxy"] = "http://127.0.0.1:7890"
+
+
+path_for_model = "./output/gpt2_openprompt/checkpoint-4500"
+
+args = {
+    "model_type": "gpt2",
+    "model_name_or_path": path_for_model,
+    "length": 80,
+    "length_penalty": 1.2,
+    "stop_token": None,
+    "temperature": 1.0,
+    "repetition_penalty": 1.2,
+    "k": 3,
+    "p": 0.9,
+    "prefix": "",
+    "padding_text": "",
+    "xlm_language": "",
+    "seed": 42,
+    "use_cpu": False,
+    "num_return_sequences": 4,
+    "fp16": False,
+    "jit": False,
+}
+
+distributed_state = PartialState(cpu=args["use_cpu"])
+
+if args["seed"] is not None:
+    set_seed(args["seed"])
+
+tokenizer = None
+model = None
+zh_en_translator = None
+nlp = None
+
+def load_model_and_components():
+    global tokenizer, model, zh_en_translator, nlp
+
+    # Initialize the model and tokenizer
+    try:
+        args["model_type"] = args["model_type"].lower()
+        model_class, tokenizer_class = MODEL_CLASSES[args["model_type"]]
+    except KeyError:
+        raise KeyError("the model {} you specified is not supported. You are welcome to add it and open a PR :)")
+
+    tokenizer = tokenizer_class.from_pretrained(args["model_name_or_path"], padding_side='left')
+    tokenizer.pad_token = tokenizer.eos_token
+    tokenizer.mask_token = tokenizer.eos_token
+    model = model_class.from_pretrained(args["model_name_or_path"])
+    print("Model loaded!")
+
+    # translator
+    zh_en_translator = Translator("Helsinki-NLP/opus-mt-zh-en")
+    print("Translator loaded!")
+
+    # filter
+    nlp = spacy.load('en_core_web_sm')
+    print("Filter loaded!")
+
+    # Set the model to the right device
+    model.to(distributed_state.device)
+
+    if args["fp16"]:
+        model.half()
+
+def chat():
+    phrase = input("Input Prompt >>")
+
+    if tokenizer is None or model is None or zh_en_translator is None or nlp is None:
+        load_model_and_components()
+
+    messages = generate_prompt(
+        prompt_text=phrase,
+        args=args,
+        zh_en_translator=zh_en_translator,
+        nlp=nlp,
+        model=model,
+        tokenizer=tokenizer,
+        distributed_state=distributed_state,
+    )
+
+    for n, m in enumerate(messages):
+        print(f"-----generated sequence {n} -----")
+        print(m)
+    print("*"*60)
+
+
+
+if __name__ == '__main__':
+    load_model_and_components()
+    while True:
+        chat()

gpt2_generation.py

@@ -2,6 +2,7 @@
 # coding=utf-8
 import inspect
 import logging
+import nltk
 from typing import Tuple
 
 import torch
@@ -261,6 +262,26 @@ class _ModelFallbackWrapper(GenerationMixin):
         return self._default._reorder_cache(past_key_values, beam_idx)
 
 
+def remove_tokens_before_copula(text):
+    sentences = text.split(",")
+    result = [sentences[0]]
+    for sentence in sentences[1:]:
+        tokens = nltk.word_tokenize(sentence)
+
+        target_indices = [i for i, token in enumerate(tokens) if token.lower() in ["is", "are", "am"]]
+
+        if target_indices:
+            last_target_index = target_indices[-1]
+            result.append(tokens[last_target_index + 1:])
+        else:
+            result.append(tokens)
+
+    all_sentences = [" ".join(sen) for sen in result[1:]]
+    all_sentences.insert(0, result[0])
+    result_text = ",".join(all_sentences)
+    return result_text
+
+
 def generate_prompt(
         prompt_text,
         args,
@@ -326,6 +347,7 @@ def generate_prompt(
                 repeat_gen_time = repeat_gen_time + 1
                 generated_sequence = model.generate(
                     input_ids=input_ids,
+                    length_penalty=args["length_penalty"],
                     max_length=args["length"] + len(encoded_prompt[0]),
                     temperature=args["temperature"],
                     top_k=args["k"],
@@ -352,13 +374,16 @@ def generate_prompt(
                     prompt_text + text[len(tokenizer.decode(encoded_prompt[0], clean_up_tokenization_spaces=True)) :]
                 )
                 # no checking for prompt_text.
-                docs = nlp(text)
-                nouns = [token.text for token in docs if token.pos_ == 'NOUN']
-                nouns = set(nouns)
-                if nouns.intersection(FORBIDDEN_NOUN) and repeat_gen_time < 10:
-                    continue
-                else:
-                    break
+                # 暂时删去关键词检测
+                # docs = nlp(text)
+                # nouns = [token.text for token in docs if token.pos_ == 'NOUN']
+                # nouns = set(nouns)
+                # if nouns.intersection(FORBIDDEN_NOUN) and repeat_gen_time < 10:
+                #     continue
+                # else:
+                #     break
+                break
+            total_sequence = remove_tokens_before_copula(total_sequence)
             generated_sequences.append(total_sequence)
 
         return generated_sequences

gpt_api.py

@@ -0,0 +1,27 @@
+import openai
+
+
+def get_response_create_data(cn_text):
+    openai.api_type = "azure"
+    openai.api_base = "https://poster-pku-gpt4.openai.azure.com/"
+    openai.api_version = "2023-07-01-preview"
+    openai.api_key = '788c2b57f1954ddc92bb27786fbcdd6e'
+
+    response = openai.ChatCompletion.create(
+        engine="dragon",
+        messages=[{"role": "system", "content": "Now you are a home improvement designer,\
+         I give you some keywords, generate a brief interior design in English, no more than words: "},
+                  {"role": "user", "content": cn_text}],
+        temperature=0.7,
+        max_tokens=800,
+        top_p=0.95,
+        frequency_penalty=0,
+        presence_penalty=0,
+        stop=None)
+    return response['choices'][0]["message"]["content"]
+
+
+if __name__ == '__main__':
+    while (1):
+        input_text = input("输入:")
+        get_response_create_data(input_text)

monitor.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+while true; do
+
+    seed=$(date +%s)
+
+    python trible.py ${seed}
+
+    if [ $? -eq 0 ]; then
+	echo "program complect, no need to restart..."
+	break
+    else
+	echo "program crash, restarting" 
+    fi
+done

sft.py

@@ -14,7 +14,7 @@ from utils import (
     get_dict_dataset,
     get_advance_dataset,)
 
-base_model = "distilgpt2"
+base_model = "gpt2"
 tokenizer, model = get_tok_and_model(f"./models/{base_model}")
 tokenizer.pad_token = tokenizer.eos_token
 rouge = evaluate.load("rouge")
@@ -53,18 +53,16 @@ print(f"data tokenize done. process time : {t2 - t1}")
 
 
 training_args = TrainingArguments(
-    output_dir=f"./output/{base_model}_openprpmpt",
+    output_dir=f"./output/{base_model}_openprompt",
     evaluation_strategy="steps",
     eval_steps=20000,
-    learning_rate=2e-5,
+    learning_rate=3e-5,
     lr_scheduler_type="constant",
     report_to="tensorboard",
     per_device_train_batch_size=64,
     per_device_eval_batch_size=32,
-    adam_beta1=0.9,
-    adam_beta2=0.98,
     save_total_limit=1,
-    num_train_epochs=80,
+    num_train_epochs=60,
     fp16=True,
     push_to_hub=False,
 )

trible.py

@@ -0,0 +1,56 @@
+import os
+import click
+import random
+import jsonlines
+
+from tqdm import tqdm
+from gpt_api import get_response_create_data
+
+
+KEYWORDS_PATH = "/data/aigc/zw/task2/pg_distilgpt/data/raw_keywords.txt"
+TARGET_PATH = "/data/aigc/zw/task2/pg_distilgpt/data/raw_discriptions.jsonl"
+
+if not os.path.exists(TARGET_PATH):
+    with open(TARGET_PATH, "w") as f:
+        pass
+
+
+def read_keywords(path=KEYWORDS_PATH):
+    
+    keywords = []
+    
+    with open(path, 'r', encoding='utf-8') as file:
+        for line in tqdm(file, desc="reading keywords"):
+            parts = line.strip().split('\t')
+            result = parts[0]
+            keywords.append(result)
+            
+    return keywords
+            
+def keywords_sampler(num, key_words):
+    random.seed()
+    while(1):
+        sampled_words = random.sample(key_words, num)
+        yield sampled_words
+            
+def create_data(keywords, total_num=10000, n=4, seed=42):
+    random.seed(seed)
+    for n, key_words in tqdm(enumerate(keywords_sampler(n, keywords)), desc="generating data"):
+        
+        res = get_response_create_data(" ".join(key_words))
+        
+        with jsonlines.open(TARGET_PATH, mode='a') as writer:
+            writer.write({"keywrods": key_words, "description": res})
+            
+        if n >= total_num:
+            print("generation data done.")
+            break
+        
+@click.command()
+@click.argument('seed', type=int)
+def main(seed):
+    keywords = read_keywords()
+    create_data(keywords, seed=seed)
+        
+if __name__ == '__main__':
+    main()