Upload lyrics_generation_rnn.py

lyrics_generation_rnn.py

@@ -0,0 +1,468 @@
+# -*- coding: utf-8 -*-
+"""lyrics_generation_rnn.ipynb
+
+Automatically generated by Colaboratory.
+
+Original file is located at
+    https://colab.research.google.com/drive/1MkBq8eqZoPqaVDczKmYhSThcV4r23z25
+"""
+
+!pip install pickle
+import pickle
+!pip install string
+import string
+
+import tensorflow as tf
+from string import punctuation
+import numpy as np
+import os
+import time
+import pickle
+model_path='/content/drive/MyDrive/Colab Notebooks'
+# create directory to store pickled files in
+if not os.path.exists(f'/content/drive/MyDrive/Colab Notebooks/pkl'):
+    os.mkdir(f'/content/drive/MyDrive/Colab Notebooks/pkl')
+
+# ----------------------------------------------------------------------
+
+### LIMITING GPU MEMORY GROWTH ###
+
+# get list of visible GPUs
+gpus = tf.config.experimental.list_physical_devices('GPU')
+
+if gpus: # if GPU(s) is detected
+    try: # try setting memory growth to true for all GPUs
+        for gpu in gpus:
+            tf.config.experimental.set_memory_growth(gpu, True) # enabling memory growth
+        logical_gpus = tf.config.experimental.list_logical_devices('GPU')
+        print('\n', len(gpus), 'Physical GPUs,', len(logical_gpus), 'Logical GPU')
+    except RuntimeError as e:
+        # memory growth must be set before GPUs have been initialized
+        print('\n', e)
+
+# ----------------------------------------------------------------------
+
+### READ IN AND CLEAN THE LYRICS DATA ###
+
+# ******TAKE IN USER INPUT FOR LYRICS (ARTIST NAME? FILE NAME?)******
+
+# read in the lyrics text file
+text = str(open('/content/drake.txt', 'r').read())
+# artist_name = input('\nPlease ')
+
+# make all letters lowercase and make line breaks into its own "word"
+words = text.lower().replace('\n', ' \n ')
+
+# remove punctuation
+for punc in punctuation:
+	words = words.replace(punc, '')
+
+# split the entire words string into a Python list of words
+words = words.split(' ')
+
+# obtain list of unique words across all lyrics
+vocab = sorted(set(words))
+print(f'\nThere are {len(vocab)} unique words in the lyrics file.')
+
+# pickle the vocab file - will need it for the generation script
+outfile = open(file='/content/drive/MyDrive/Colab Notebooks/pkl/vocab', mode='wb')
+pickle.dump(vocab, outfile)
+outfile.close()
+
+# ----------------------------------------------------------------------
+
+### WORD MAPPING ###
+
+# map unique characters to indices
+word2idx = {u:i for i, u in enumerate(vocab)}
+
+# pickle this since it is needed in text generation
+outfile = open(file='/content/drive/MyDrive/Colab Notebooks/pkl/word2idx', mode='wb')
+pickle.dump(word2idx, outfile)
+outfile.close()
+
+# reverse the map - use this to specify an index to obtain a character
+idx2word = np.array(vocab)
+
+# pickle this since it is needed in text generation
+outfile = open(file='/content/drive/MyDrive/Colab Notebooks/pkl/idx2word', mode='wb')
+pickle.dump(idx2word, outfile)
+outfile.close()
+
+# entire text document represented in the above character-to-indices mapping
+words_as_int = np.array([word2idx[c] for c in words])
+
+# ----------------------------------------------------------------------
+
+### CREATING TRAINING EXAMPLES & TARGETS ###
+
+# ******TAKE IN USER INPUT FOR SEQUENCE LENGTH?******
+
+# max sentence length (in number of words) desired for training
+seq_length = 100
+# seq_length = input('\nPlease enter a desired sequence length (in number of words) to train the model on: ')
+examples_per_epoch = len(words) // (seq_length + 1)
+
+# create training examples/targets
+word_dataset = tf.data.Dataset.from_tensor_slices(words_as_int)
+
+# data type of train examples/targets
+print('\n', type(word_dataset))
+
+# create sequence batches from the word_dataset
+sequences = word_dataset.batch(seq_length + 1, drop_remainder=True)
+print('\n', type(sequences))
+
+# define the shifting (splitting) function
+def split_input_target(chunk):
+    input_text = chunk[:-1] # up to but not including the last character
+    target_text = chunk[1:] # everything except for the firs tcharacter
+    return input_text, target_text
+
+# apply the shifting to create input texts and target texts that comprise of our dataset
+dataset = sequences.map(split_input_target)
+
+# ----------------------------------------------------------------------
+
+### CREATE TRAINING BATCHES ###
+
+# batch size
+BATCH_SIZE = 64
+
+# buffer size to shuffle the dataset
+# (TensorFlow data is designed to work with possibly infinite sequences,
+# so it doesn't attempt to shuffle the entire sequence in memory.  Instead,
+# it maintains a buffer in which it shuffles elements)
+BUFFER_SIZE = 10000
+
+# create a dataset that has been shuffled and batched
+dataset_sb = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)
+
+# display batch dataset shapes and data types
+print('\n', dataset_sb)
+
+# ----------------------------------------------------------------------
+
+### BUILDING THE RNN ###
+
+# vocabulary length (number of unique words in dataset)
+vocab_size = len(vocab)
+
+# embedding dimension
+embedding_dim = 256
+
+# number of RNN units
+rnn_units = 1024
+
+# pickle model parameters - will need in the generation script
+model_params = [vocab_size, embedding_dim, rnn_units]
+outfile = open(file='/content/drive/MyDrive/Colab Notebooks/pkl/model_params', mode='wb')
+pickle.dump(model_params, outfile)
+outfile.close()
+
+# helper function to quickly build the RNN model based on vocab size, embedding dimension, number of RNN units, and batch size
+def build_model(vocab_size, embedding_dim, rnn_units, batch_size):
+
+	# initialize sequential model architecture
+    model = tf.keras.Sequential()
+
+    # add embedding layer
+    model.add(tf.keras.layers.Embedding(
+        input_dim = vocab_size,
+        output_dim = embedding_dim,
+        batch_input_shape=[batch_size, None]
+    ))
+
+    # add recurrent layer
+    model.add(tf.keras.layers.GRU(
+        units = rnn_units,
+        return_sequences = True,
+        stateful = True,
+        recurrent_initializer = 'glorot_uniform'
+    ))
+
+    # add dense layer
+    model.add(tf.keras.layers.Dense(units=vocab_size))
+    model_path= '/content/drive/MyDrive/Colab Notebooks'
+
+    def save_model(self, model_path):
+        # Save the model weights
+        self.save_weights(model_path)
+        print(f"Model saved to {model_path}")
+    return model
+
+# build the model using the above helper function
+rnn = build_model(
+    vocab_size = vocab_size,
+    embedding_dim = embedding_dim,
+    rnn_units = rnn_units,
+    batch_size = BATCH_SIZE
+)
+
+# check the shape of the output
+for input_example_batch, target_example_batch in dataset_sb.take(1):
+    example_batch_predictions = rnn(input_example_batch)
+    print('\n', example_batch_predictions.shape, '# (batch_size, sequence_length, vocab_size)')
+
+# model architecture summary
+print('\n', rnn.summary(), '\n')
+
+# ----------------------------------------------------------------------
+
+### SET UP METRICS ###
+
+# helper function to obtain the loss function
+def loss(labels, logits):
+    return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)
+
+# compile the model
+rnn.compile(
+    optimizer = 'adam',
+    loss = loss,
+    metrics = ['accuracy']
+)
+
+# create directory where the checkpoints will be saved
+checkpoint_dir = '/content/drive/MyDrive/Colab Notebooks/training_checkpoints'
+
+# name of the checkpoint files
+checkpoint_prefix = os.path.join(checkpoint_dir, 'checkpoint')
+
+# create checkpoints-saving object
+checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
+    filepath = checkpoint_prefix,
+    monitor = 'loss',
+    save_best_only = True,
+    mode = 'min',
+    save_weights_only = True
+)
+
+# ----------------------------------------------------------------------
+
+### MODEL TRAINING ###
+
+# set number of desired epochs
+EPOCHS = 200
+
+# training!
+history = rnn.fit(
+    x = dataset_sb,
+    epochs = EPOCHS,
+    callbacks = [checkpoint_callback]
+)
+
+build_model.save('/content/drive/MyDrive/Colab Notebooks')
+
+import tensorflow as tf
+from string import punctuation
+import pickle
+
+# ----------------------------------------------------------------------
+
+### LIMITING GPU MEMORY GROWTH ###
+
+# get list of visible GPUs
+gpus = tf.config.experimental.list_physical_devices('GPU')
+
+if gpus: # if GPU(s) is detected
+    try: # try setting memory growth to true for all GPUs
+        for gpu in gpus:
+            tf.config.experimental.set_memory_growth(gpu, True) # enabling memory growth
+        logical_gpus = tf.config.experimental.list_logical_devices('GPU')
+        print('\n', len(gpus), 'Physical GPUs,', len(logical_gpus), 'Logical GPU')
+    except RuntimeError as e:
+        # memory growth must be set before GPUs have been initialized
+        print('\n', e)
+
+# -------------------------------------------------------------------------
+
+### MODEL BUILDING FUNCTION FROM TRAINING SCRIPT ###
+
+# helper function to quickly build the RNN model based on vocab size, embedding dimension, number of RNN units, and batch size
+def build_model(vocab_size, embedding_dim, rnn_units, batch_size):
+    model = tf.keras.Sequential()
+
+    model.add(tf.keras.layers.Embedding(
+        input_dim = vocab_size,
+        output_dim = embedding_dim,
+        batch_input_shape=[batch_size, None]
+    ))
+
+    model.add(tf.keras.layers.GRU(
+        units = rnn_units,
+        return_sequences = True,
+        stateful = True,
+        recurrent_initializer = 'glorot_uniform'
+    ))
+
+    model.add(tf.keras.layers.Dense(units=vocab_size))
+
+    model_path= '/content/drive/MyDrive/Colab Notebooks'
+
+    def save_model(self, model_path):
+        # Save the model weights
+        self.save_weights(model_path)
+        print(f"Model saved to {model_path}")
+
+    return model
+
+# -------------------------------------------------------------------------
+
+### INITIATE MODEL AND LOAD IN WEIGHTS FROM CHECKPOINT ###
+
+# unpickle the model parameters from the training script
+infile = open(file='pkl/model_params', mode='rb')
+vocab_size, embedding_dim, rnn_units = pickle.load(infile)
+infile.close()
+
+# initiate new model instance
+rnn_cp = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)
+
+# load saved weights from checkpoint into new model instance
+rnn_cp.load_weights(tf.train.latest_checkpoint('./training_checkpoints'))
+
+# build the model with a new input shape
+rnn_cp.build(tf.TensorShape([1, None]))
+
+# -------------------------------------------------------------------------
+
+### TEXT PREDICTION FUNCTION ###
+
+# unpickle the index-word files that were pickled from the training script
+infile = open(file='pkl/word2idx', mode='rb')
+word2idx = pickle.load(infile)
+infile.close()
+infile = open(file='pkl/idx2word', mode='rb')
+idx2word = pickle.load(infile)
+infile.close()
+#build_model.is_valid():
+#build_model.save('/content/drive/MyDrive/Colab Notebooks')
+
+def generate_text(model, start_string, num_generate=500, temperature=1.0):
+
+    # num of chars to generate
+    num_generate = num_generate
+
+    # vectorizing the start string to numbers
+    input_eval = [word2idx[s] for s in start_string]
+    input_eval = tf.expand_dims(input=input_eval, axis=0) # returns a tensor with a length-1 axis inserted at index `axis`
+
+    # empty string to store results
+    text_generated = list()
+
+    # "temperature"
+    # low temperature results in more predictable text,
+    # high temperature results in more surprising text.
+    # feel free to experiment with this parameter
+    temperature = 1.0
+
+    # the batch size was defined when we loaded model weights from training
+
+    model.reset_states()
+    for i in range(num_generate):
+        predictions = model(input_eval)
+
+        # remove the batch dimension
+        predictions = tf.squeeze(predictions, 0)
+
+        # use a categorical distribution to predict the character returned by the model
+        preidctions = predictions / temperature
+        predicted_id = tf.random.categorical(predictions, num_samples=1)[-1, 0].numpy()
+
+        # pass the predicted character as the next input to the model along with the previous hidden state
+        input_eval = tf.expand_dims([predicted_id], 0)
+
+        text_generated.append(idx2word[predicted_id])
+
+    return(' '.join(start_string + text_generated))
+
+# -------------------------------------------------------------------------
+
+### TAKE IN INPUT STRING AND CHECK IF ALL WORDS IN IT ARE IN THE VOCABULARY ###
+# (this is a requirement for text generation)
+
+# unpickle the vocabulary file that was pickled from the training script
+infile = open(file='pkl/vocab', mode='rb')
+vocab = pickle.load(infile)
+infile.close()
+
+# initialize the checking loop
+check = True
+
+while check:
+
+	# take in user input for starting lyrics
+	start_string = input('\nPlease input some text to initiate the lyrics generation (caps insensitive):\n')
+
+	# lowercase
+	start_string = start_string.lower()
+
+	# remove punctuation
+	for punc in punctuation:
+		start_string = start_string.replace(punc, '')
+
+	# create a list where each element is one word from the start string
+	start_string = start_string.split(' ')
+
+	# store all words that aren't in the vocabulary
+	non_vocab = []
+
+	# for every word in the start string
+	for word in start_string:
+
+		# if the word is NOT in the vocabulary
+		if word not in vocab:
+
+			# add the word to the non_vocab variable
+			non_vocab.append(word)
+
+	# if the non-vocab list is empty (i.e. all words in the start string are in the vocab)
+	if non_vocab == []:
+
+		# break out of the loop
+		check = False
+
+	# if there are words not in the vocabulary
+	else:
+
+		# print what those words are
+		print(f'\nWords in the input text not present in the vocabulary are: {", ".join(non_vocab)}')
+		print('\nAll input words must be in the vocabulary.')
+
+# -------------------------------------------------------------------------
+
+### TEXT GENERATION ###
+
+# text generation!
+print('\n', generate_text(rnn_cp, start_string=start_string, num_generate=250))
+
+### SAVE TO FILE??? ###
+
+# -------------------------------------------------------------------------
+
+# -------------------------------------------------------------------------
+
+
+
+build_model.save('/content/drive/MyDrive/Colab Notebooks')
+
+model = build_model
+
+"""import tensorflow as tf
+build_model.state_dict()
+# Assuming you have a trained model named 'model'
+model = ...
+
+# Define the path to save the model
+model_path = 'path_to_save_model'
+
+# Save the entire model (architecture, weights, and optimizer state)
+model.save(model_path)
+
+[link text](https:// [link text](https://))# Alternatively, you can save only the model weights
+model.save_weights('path_to_save_weights')
+
+# You can also save the model in a format optimized for serving
+tf.saved_model.save(model, 'path_for_serving')
+
+"""