upl base

.gitattributes

@@ -1,35 +1,36 @@
 *.7z filter=lfs diff=lfs merge=lfs -text
 *.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
 *.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
 *.ftz filter=lfs diff=lfs merge=lfs -text
 *.gz filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
 *.ot filter=lfs diff=lfs merge=lfs -text
 *.parquet filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*.tfevents* filter=lfs diff=lfs merge=lfs -text
+*.db* filter=lfs diff=lfs merge=lfs -text
+*.ark* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.sf3 filter=lfs diff=lfs merge=lfs -text
+*.AppImage filter=lfs diff=lfs merge=lfs -textlibnss3.so filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,8 @@
+__pycache__/*
+output/*
+rename.sh
+test.py
+gpt2-abcmusic/*
+*.pth
+tmp/*
+mscore3/*

README.md

@@ -1,10 +1,10 @@
 ---
 title: HoyoGPT
-emoji: 🐢
-colorFrom: pink
-colorTo: purple
+emoji: 🎹
+colorFrom: green
+colorTo: pink
 sdk: gradio
-sdk_version: 4.12.0
+sdk_version: 4.7.1
 app_file: app.py
 pinned: false
 license: mit

app.py

@@ -0,0 +1,215 @@
+import re
+import os
+import time
+import torch
+import shutil
+import argparse
+import gradio as gr
+from utils import *
+from config import *
+from render import *
+from music21 import converter
+from transformers import GPT2Config
+import warnings
+warnings.filterwarnings('ignore')
+
+
+def abc_to_midi(abc_content, output_midi_path):
+    # 解析 ABC 格式的乐谱
+    score = converter.parse(abc_content)
+
+    # 将乐谱保存为 MIDI 文件
+    score.write('midi', fp=output_midi_path)
+    return output_midi_path
+
+
+def get_args(parser):
+    parser.add_argument('-num_tunes', type=int, default=1,
+                        help='the number of independently computed returned tunes')
+    parser.add_argument('-max_patch', type=int, default=128,
+                        help='integer to define the maximum length in tokens of each tune')
+    parser.add_argument('-top_p', type=float, default=0.8,
+                        help='float to define the tokens that are within the sample operation of text generation')
+    parser.add_argument('-top_k', type=int, default=8,
+                        help='integer to define the tokens that are within the sample operation of text generation')
+    parser.add_argument('-temperature', type=float, default=1.2,
+                        help='the temperature of the sampling operation')
+    parser.add_argument('-seed', type=int, default=None,
+                        help='seed for randomstate')
+    parser.add_argument('-show_control_code', type=bool,
+                        default=True, help='whether to show control code')
+    args = parser.parse_args()
+
+    return args
+
+
+def generate_abc(args, region):
+    patchilizer = Patchilizer()
+
+    patch_config = GPT2Config(
+        num_hidden_layers=PATCH_NUM_LAYERS,
+        max_length=PATCH_LENGTH,
+        max_position_embeddings=PATCH_LENGTH,
+        vocab_size=1
+    )
+
+    char_config = GPT2Config(
+        num_hidden_layers=CHAR_NUM_LAYERS,
+        max_length=PATCH_SIZE,
+        max_position_embeddings=PATCH_SIZE,
+        vocab_size=128
+    )
+
+    model = TunesFormer(patch_config, char_config, share_weights=SHARE_WEIGHTS)
+
+    filename = WEIGHT_PATH
+
+    if os.path.exists(filename):
+        print(f"Weights already exist at '{filename}'. Loading...")
+
+    else:
+        download()
+
+    checkpoint = torch.load(filename, map_location=torch.device('cpu'))
+    model.load_state_dict(checkpoint['model'])
+    model = model.to(device)
+    model.eval()
+
+    prompt = template(region)
+
+    tunes = ""
+    num_tunes = args.num_tunes
+    max_patch = args.max_patch
+    top_p = args.top_p
+    top_k = args.top_k
+    temperature = args.temperature
+    seed = args.seed
+    show_control_code = args.show_control_code
+
+    print(" HYPERPARAMETERS ".center(60, "#"), '\n')
+    args = vars(args)
+
+    for key in args.keys():
+        print(f'{key}: {str(args[key])}')
+
+    print('\n', " OUTPUT TUNES ".center(60, "#"))
+
+    start_time = time.time()
+
+    for i in range(num_tunes):
+        tune = f"X:{str(i + 1)}\n{prompt}"
+        lines = re.split(r'(\n)', tune)
+        tune = ""
+        skip = False
+        for line in lines:
+            if show_control_code or line[:2] not in ["S:", "B:", "E:"]:
+                if not skip:
+                    print(line, end="")
+                    tune += line
+
+                skip = False
+
+            else:
+                skip = True
+
+        input_patches = torch.tensor(
+            [patchilizer.encode(prompt, add_special_patches=True)[:-1]],
+            device=device
+        )
+
+        if tune == "":
+            tokens = None
+
+        else:
+            prefix = patchilizer.decode(input_patches[0])
+            remaining_tokens = prompt[len(prefix):]
+            tokens = torch.tensor(
+                [patchilizer.bos_token_id]+[ord(c) for c in remaining_tokens],
+                device=device
+            )
+
+        while input_patches.shape[1] < max_patch:
+            predicted_patch, seed = model.generate(
+                input_patches,
+                tokens,
+                top_p=top_p,
+                top_k=top_k,
+                temperature=temperature,
+                seed=seed
+            )
+            tokens = None
+
+            if predicted_patch[0] != patchilizer.eos_token_id:
+                next_bar = patchilizer.decode([predicted_patch])
+
+                if show_control_code or next_bar[:2] not in ["S:", "B:", "E:"]:
+                    print(next_bar, end="")
+                    tune += next_bar
+
+                if next_bar == "":
+                    break
+
+                next_bar = remaining_tokens+next_bar
+                remaining_tokens = ""
+
+                predicted_patch = torch.tensor(
+                    patchilizer.bar2patch(next_bar),
+                    device=device
+                ).unsqueeze(0)
+
+                input_patches = torch.cat(
+                    [input_patches, predicted_patch.unsqueeze(0)],
+                    dim=1
+                )
+
+            else:
+                break
+
+        tunes += f"{tune}\n\n"
+        print("\n")
+
+    print("Generation time: {:.2f} seconds".format(time.time() - start_time))
+    create_dir('./tmp')
+    timestamp = time.strftime("%a_%d_%b_%Y_%H_%M_%S", time.localtime())
+    out_midi = abc_to_midi(tunes, f'./tmp/[{region}]{timestamp}.mid')
+    add_path()
+    png_file = midi2png(out_midi)
+    wav_file = midi2wav(out_midi)
+
+    return tunes, out_midi, png_file, wav_file
+
+
+def inference(region):
+    if os.path.exists('./tmp'):
+        shutil.rmtree('./tmp')
+
+    parser = argparse.ArgumentParser()
+    args = get_args(parser)
+    return generate_abc(args, region)
+
+
+with gr.Blocks() as demo:
+    with gr.Row():
+        with gr.Column():
+            region_opt = gr.Dropdown(
+                choices=[
+                    'Mondstadt', 'Liyue', 'Inazuma', 'Sumeru', 'Fontaine'
+                ],
+                value='Liyue',
+                label='Region'
+            )
+            gen_btn = gr.Button("Generate")
+
+        with gr.Column():
+            wav_output = gr.Audio(label='Audio', type='filepath')
+            dld_midi = gr.components.File(label="Download MIDI")
+            abc_output = gr.TextArea(label='abc score')
+            img_score = gr.Image(label='Staff', type='filepath')
+
+    gen_btn.click(
+        inference,
+        inputs=region_opt,
+        outputs=[abc_output, dld_midi, img_score, wav_output]
+    )
+
+demo.launch(share=True)

conda.txt

@@ -0,0 +1,5 @@
+python=3.10
+pytorch=1.12.1
+torchvision=0.13.1
+torchaudio=0.12.1
+cudatoolkit=11.3.1

config.py

@@ -0,0 +1,19 @@
+PATCH_LENGTH = 128      # Patch Length
+PATCH_SIZE = 32       # Patch Size
+
+PATCH_NUM_LAYERS = 9         # Number of layers in the encoder
+CHAR_NUM_LAYERS = 3          # Number of layers in the decoder
+
+# Number of epochs to train for (if early stopping doesn't intervene)
+NUM_EPOCHS = 5  # 32
+LEARNING_RATE = 5e-5            # Learning rate for the optimizer
+# Batch size for patch during training, 0 for full context
+PATCH_SAMPLING_BATCH_SIZE = 0
+LOAD_FROM_CHECKPOINT = True     # Whether to load weights from a checkpoint
+# Whether to share weights between the encoder and decoder
+SHARE_WEIGHTS = False
+WEIGHT_URL = 'https://huggingface.co/MuGeminorum/hoyoGPT/resolve/main/weights.pth'
+ZH_WEIGHT_URL = 'https://www.modelscope.cn/api/v1/models/MuGeminorum/hoyoGPT/repo?Revision=master&FilePath=weights.pth'
+WEIGHT_PATH = 'weights.pth'
+LOG_PATH = 'logs.txt'
+PROMPT_PATH = 'prompt.txt'

render.py

@@ -0,0 +1,73 @@
+import os
+import sys
+import subprocess
+from PIL import Image
+from pdf2image import convert_from_path
+from utils import download
+
+
+def add_path():
+    # """
+    # 将指定目录添加到 LD_LIBRARY_PATH 环境变量中，并在当前 Python 进程中生效。
+
+    # Parameters:
+    # - directory_path (str): 要添加的目录路径。
+    # """
+    # dir_path = os.path.join(os.getcwd(), 'lib')
+    # # 获取当前环境变量的值
+    # current_path = os.environ.get("LD_LIBRARY_PATH", "")
+
+    # # 将目录路径添加到 LD_LIBRARY_PATH 中
+    # new_path = f"{current_path}:{dir_path}"
+
+    # # 设置 LD_LIBRARY_PATH 环境变量，以便在当前 Python 进程中生效
+    # os.environ["LD_LIBRARY_PATH"] = new_path
+    os.environ['QT_QPA_PLATFORM'] = 'offscreen'
+
+
+if sys.platform.startswith('linux'):
+    apkname = 'MuseScore.AppImage'
+    extra_dir = 'squashfs-root'
+    download(
+        filename=apkname,
+        url='https://cdn.jsdelivr.net/musescore/v4.2.0/MuseScore-4.2.0.233521125-x86_64.AppImage'
+    )
+    if not os.path.exists(extra_dir):
+        subprocess.run(['chmod', '+x', f'./{apkname}'])
+        subprocess.run([f'./{apkname}', '--appimage-extract'])
+
+    mscore = f'./{extra_dir}/AppRun'
+
+else:
+    mscore = "D:/Program Files/MuseScore 3/bin/MuseScore3.exe"
+
+
+def midi2wav(mid_file: str):
+    wav_file = mid_file.replace('.mid', '.wav')
+    command = [mscore, "-o", wav_file, mid_file]
+    result = subprocess.run(command)
+    print(result)
+    return wav_file
+
+
+def pdf_to_img(pdf_path: str):
+    output_path = pdf_path.replace('.pdf', '.jpg')
+    images = convert_from_path(pdf_path)
+    combined_image = Image.new(
+        'RGB', (images[0].width, sum(image.height for image in images))
+    )
+    y_offset = 0
+    for image in images:
+        combined_image.paste(image, (0, y_offset))
+        y_offset += image.height
+
+    combined_image.save(output_path)
+    return output_path
+
+
+def midi2png(mid_file: str):
+    pdf_score = mid_file.replace('.mid', '.pdf')
+    command = [mscore, "-o", pdf_score, mid_file]
+    result = subprocess.run(command)
+    print(result)
+    return pdf_to_img(pdf_score)

requirements.txt

@@ -0,0 +1,9 @@
+transformers==4.18.0
+samplings==0.1.7
+unidecode
+music21
+autopep8
+pillow==9.4.0
+gradio
+pdf2image
+torch

utils.py

@@ -0,0 +1,388 @@
+import os
+import re
+import torch
+import random
+from config import *
+from tqdm import tqdm
+from unidecode import unidecode
+from torch.utils.data import Dataset
+from transformers import GPT2Model, GPT2LMHeadModel, PreTrainedModel
+from samplings import top_p_sampling, top_k_sampling, temperature_sampling
+
+if torch.cuda.is_available():
+    device = torch.device("cuda")
+else:
+    device = torch.device("cpu")
+
+
+def template(region):
+    return f'''A:{region}
+S:2
+B:9
+E:4
+B:9
+L:1/8
+M:3/4
+K:D
+de |"D"'''
+
+
+def create_dir(dir_path):
+    if not os.path.exists(dir_path):
+        os.makedirs(dir_path)
+
+
+def download(filename=WEIGHT_PATH, url=WEIGHT_URL):
+    import time
+    import requests
+    try:
+        response = requests.get(url, stream=True)
+        total_size = int(response.headers.get('content-length', 0))
+        chunk_size = 1024
+
+        with open(filename, 'wb') as file, tqdm(
+            desc=f"Downloading weights to '{filename}'...",
+            total=total_size,
+            unit='B',
+            unit_scale=True,
+            unit_divisor=1024,
+        ) as bar:
+            for data in response.iter_content(chunk_size=chunk_size):
+                size = file.write(data)
+                bar.update(size)
+
+    except ConnectionError as e:
+        print(f"Error: {e}")
+        time.sleep(3)
+        download(filename, ZH_WEIGHT_URL)
+
+
+class Patchilizer:
+    """
+    A class for converting music bars to patches and vice versa. 
+    """
+
+    def __init__(self):
+        self.delimiters = ["|:", "::", ":|", "[|", "||", "|]", "|"]
+        self.regexPattern = f"({'|'.join(map(re.escape, self.delimiters))})"
+        self.pad_token_id = 0
+        self.bos_token_id = 1
+        self.eos_token_id = 2
+
+    def split_bars(self, body):
+        """
+        Split a body of music into individual bars.
+        """
+        bars = re.split(self.regexPattern, ''.join(body))
+        bars = list(filter(None, bars))
+        # remove empty strings
+        if bars[0] in self.delimiters:
+            bars[1] = bars[0] + bars[1]
+            bars = bars[1:]
+
+        bars = [bars[i * 2] + bars[i * 2 + 1] for i in range(len(bars) // 2)]
+        return bars
+
+    def bar2patch(self, bar, patch_size=PATCH_SIZE):
+        """
+        Convert a bar into a patch of specified length.
+        """
+        patch = [self.bos_token_id] + \
+            [ord(c) for c in bar] + [self.eos_token_id]
+        patch = patch[:patch_size]
+        patch += [self.pad_token_id] * (patch_size - len(patch))
+        return patch
+
+    def patch2bar(self, patch):
+        """
+        Convert a patch into a bar.
+        """
+        return ''.join(chr(idx) if idx > self.eos_token_id else '' for idx in patch if idx != self.eos_token_id)
+
+    def encode(self, abc_code, patch_length=PATCH_LENGTH, patch_size=PATCH_SIZE, add_special_patches=False):
+        """
+        Encode music into patches of specified length.
+        """
+        lines = unidecode(abc_code).split('\n')
+        lines = list(filter(None, lines))  # remove empty lines
+
+        body = ""
+        patches = []
+
+        for line in lines:
+            if len(line) > 1 and ((line[0].isalpha() and line[1] == ':') or line.startswith('%%score')):
+                if body:
+                    bars = self.split_bars(body)
+                    patches.extend(
+                        self.bar2patch(bar + '\n' if idx == len(bars) - 1 else bar, patch_size) for idx, bar in enumerate(bars)
+                    )
+                    body = ""
+
+                patches.append(self.bar2patch(line + '\n', patch_size))
+
+            else:
+                body += line + '\n'
+
+        if body:
+            patches.extend(
+                self.bar2patch(bar, patch_size) for bar in self.split_bars(body)
+            )
+
+        if add_special_patches:
+            bos_patch = [self.bos_token_id] * \
+                (patch_size-1) + [self.eos_token_id]
+            eos_patch = [self.bos_token_id] + \
+                [self.eos_token_id] * (patch_size-1)
+            patches = [bos_patch] + patches + [eos_patch]
+
+        return patches[:patch_length]
+
+    def decode(self, patches):
+        """
+        Decode patches into music.
+        """
+        return ''.join(self.patch2bar(patch) for patch in patches)
+
+
+class PatchLevelDecoder(PreTrainedModel):
+    """
+    An Patch-level Decoder model for generating patch features in an auto-regressive manner. 
+    It inherits PreTrainedModel from transformers.
+    """
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.patch_embedding = torch.nn.Linear(PATCH_SIZE * 128, config.n_embd)
+        torch.nn.init.normal_(self.patch_embedding.weight, std=0.02)
+        self.base = GPT2Model(config)
+
+    def forward(self, patches: torch.Tensor) -> torch.Tensor:
+        """
+        The forward pass of the patch-level decoder model.
+        :param patches: the patches to be encoded
+        :return: the encoded patches
+        """
+        patches = torch.nn.functional.one_hot(patches, num_classes=128).float()
+        patches = patches.reshape(len(patches), -1, PATCH_SIZE * 128)
+        patches = self.patch_embedding(patches.to(self.device))
+
+        return self.base(inputs_embeds=patches)
+
+
+class CharLevelDecoder(PreTrainedModel):
+    """
+    A Char-level Decoder model for generating the characters within each bar patch sequentially. 
+    It inherits PreTrainedModel from transformers.
+    """
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.pad_token_id = 0
+        self.bos_token_id = 1
+        self.eos_token_id = 2
+        self.base = GPT2LMHeadModel(config)
+
+    def forward(self, encoded_patches: torch.Tensor, target_patches: torch.Tensor, patch_sampling_batch_size: int):
+        """
+        The forward pass of the char-level decoder model.
+        :param encoded_patches: the encoded patches
+        :param target_patches: the target patches
+        :return: the decoded patches
+        """
+        # preparing the labels for model training
+        target_masks = target_patches == self.pad_token_id
+        labels = target_patches.clone().masked_fill_(target_masks, -100)
+
+        # masking the labels for model training
+        target_masks = torch.ones_like(labels)
+        target_masks = target_masks.masked_fill_(labels == -100, 0)
+
+        # select patches
+        if patch_sampling_batch_size != 0 and patch_sampling_batch_size < target_patches.shape[0]:
+            indices = list(range(len(target_patches)))
+            random.shuffle(indices)
+            selected_indices = sorted(indices[:patch_sampling_batch_size])
+
+            target_patches = target_patches[selected_indices, :]
+            target_masks = target_masks[selected_indices, :]
+            encoded_patches = encoded_patches[selected_indices, :]
+            labels = labels[selected_indices, :]
+
+        # get input embeddings
+        inputs_embeds = torch.nn.functional.embedding(
+            target_patches,
+            self.base.transformer.wte.weight
+        )
+
+        # concatenate the encoded patches with the input embeddings
+        inputs_embeds = torch.cat(
+            (encoded_patches.unsqueeze(1), inputs_embeds[:, 1:, :]),
+            dim=1
+        )
+
+        return self.base(
+            inputs_embeds=inputs_embeds,
+            attention_mask=target_masks,
+            labels=labels
+        )
+
+    def generate(self, encoded_patch: torch.Tensor, tokens: torch.Tensor):
+        """
+        The generate function for generating a patch based on the encoded patch and already generated tokens.
+        :param encoded_patch: the encoded patch
+        :param tokens: already generated tokens in the patch
+        :return: the probability distribution of next token
+        """
+        encoded_patch = encoded_patch.reshape(1, 1, -1)
+        tokens = tokens.reshape(1, -1)
+
+        # Get input embeddings
+        tokens = torch.nn.functional.embedding(
+            tokens,
+            self.base.transformer.wte.weight
+        )
+
+        # Concatenate the encoded patch with the input embeddings
+        tokens = torch.cat((encoded_patch, tokens[:, 1:, :]), dim=1)
+
+        # Get output from model
+        outputs = self.base(inputs_embeds=tokens)
+
+        # Get probabilities of next token
+        probs = torch.nn.functional.softmax(
+            outputs.logits.squeeze(0)[-1],
+            dim=-1
+        )
+
+        return probs
+
+
+class TunesFormer(PreTrainedModel):
+    """
+    TunesFormer is a hierarchical music generation model based on bar patching. 
+    It includes a patch-level decoder and a character-level decoder.
+    It inherits PreTrainedModel from transformers.
+    """
+
+    def __init__(self, encoder_config, decoder_config, share_weights=False):
+        super().__init__(encoder_config)
+        self.pad_token_id = 0
+        self.bos_token_id = 1
+        self.eos_token_id = 2
+        if share_weights:
+            max_layers = max(
+                encoder_config.num_hidden_layers,
+                decoder_config.num_hidden_layers
+            )
+
+            max_context_size = max(
+                encoder_config.max_length,
+                decoder_config.max_length
+            )
+
+            max_position_embeddings = max(
+                encoder_config.max_position_embeddings,
+                decoder_config.max_position_embeddings
+            )
+
+            encoder_config.num_hidden_layers = max_layers
+            encoder_config.max_length = max_context_size
+            encoder_config.max_position_embeddings = max_position_embeddings
+            decoder_config.num_hidden_layers = max_layers
+            decoder_config.max_length = max_context_size
+            decoder_config.max_position_embeddings = max_position_embeddings
+
+        self.patch_level_decoder = PatchLevelDecoder(encoder_config)
+        self.char_level_decoder = CharLevelDecoder(decoder_config)
+
+        if share_weights:
+            self.patch_level_decoder.base = self.char_level_decoder.base.transformer
+
+    def forward(self, patches: torch.Tensor, patch_sampling_batch_size: int = PATCH_SAMPLING_BATCH_SIZE):
+        """
+        The forward pass of the TunesFormer model.
+        :param patches: the patches to be both encoded and decoded
+        :return: the decoded patches
+        """
+        patches = patches.reshape(len(patches), -1, PATCH_SIZE)
+        encoded_patches = self.patch_level_decoder(
+            patches)["last_hidden_state"]
+
+        return self.char_level_decoder(encoded_patches.squeeze(0)[:-1, :], patches.squeeze(0)[1:, :], patch_sampling_batch_size)
+
+    def generate(
+        self,
+        patches: torch.Tensor,
+        tokens: torch.Tensor,
+        top_p: float = 1,
+        top_k: int = 0,
+        temperature: float = 1,
+        seed: int = None
+    ):
+        """
+        The generate function for generating patches based on patches.
+        :param patches: the patches to be encoded
+        :return: the generated patches
+        """
+        patches = patches.reshape(len(patches), -1, PATCH_SIZE)
+        encoded_patches = self.patch_level_decoder(
+            patches)["last_hidden_state"]
+
+        if tokens == None:
+            tokens = torch.tensor([self.bos_token_id], device=self.device)
+
+        generated_patch = []
+        random.seed(seed)
+
+        while True:
+            if seed != None:
+                n_seed = random.randint(0, 1000000)
+                random.seed(n_seed)
+
+            else:
+                n_seed = None
+
+            prob = self.char_level_decoder.generate(
+                encoded_patches[0][-1],
+                tokens
+            ).cpu().detach().numpy()
+
+            prob = top_p_sampling(prob, top_p=top_p, return_probs=True)
+            prob = top_k_sampling(prob, top_k=top_k, return_probs=True)
+
+            token = temperature_sampling(
+                prob,
+                temperature=temperature,
+                seed=n_seed
+            )
+
+            generated_patch.append(token)
+            if token == self.eos_token_id or len(tokens) >= PATCH_SIZE - 1:
+                break
+
+            else:
+                tokens = torch.cat(
+                    (tokens, torch.tensor([token], device=self.device)),
+                    dim=0
+                )
+
+        return generated_patch, n_seed
+
+
+class PatchilizedData(Dataset):
+    def __init__(self, items, patchilizer):
+        self.texts = []
+
+        for item in tqdm(items):
+            text = item['control code'] + \
+                "\n".join(item['abc notation'].split('\n')[1:])
+            input_patch = patchilizer.encode(text, add_special_patches=True)
+            input_patch = torch.tensor(input_patch)
+            if torch.sum(input_patch) != 0:
+                self.texts.append(input_patch)
+
+    def __len__(self):
+        return len(self.texts)
+
+    def __getitem__(self, idx):
+        return self.texts[idx]