File size: 11,850 Bytes
5903239 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 |
{
"cells": [
{
"cell_type": "code",
"execution_count": 2,
"id": "55c95870",
"metadata": {},
"outputs": [],
"source": [
"from e621_utilities import construct_text_description_from_json_entry\n",
"import json\n",
"from math import log\n",
"import random\n",
"import numpy as np\n",
"from collections import Counter\n",
"\n",
"\n",
"IMAGE_COUNT=None\n",
"INPUT_JSONS=['D:/PythonExperiments/e621_high_score.json','D:/PythonExperiments/e621_low_score.json']\n",
"\n",
"\n",
"def score_post_log_favs(post):\n",
" return min(1.0, (log(int(post['fav_count'])+1) / 10))\n",
"\n",
"def load_tag_sets(data_list):\n",
" scores = []\n",
" text_descriptions = []\n",
" artists = []\n",
" for data in data_list:\n",
" text_description = construct_text_description_from_json_entry(data)\n",
" artist, text_description = extract_artist(text_description)\n",
" \n",
" score =score_post_log_favs(data)\n",
" score_int = round(score * 10)\n",
" text_description.append(f\"score:{score_int}\")\n",
" \n",
" text_descriptions.append(text_description)\n",
" artists.append(artist)\n",
" return text_descriptions, artists\n",
"\n",
"def load_data(input_json):\n",
" with open(input_json) as f:\n",
" data_list = json.load(f)[:IMAGE_COUNT] \n",
" # Load scores and tag sets from regular Python variables\n",
" return load_tag_sets(data_list)\n",
"\n",
"def extract_artist(tags):\n",
" for tag in tags:\n",
" if tag.startswith('by '):\n",
" tags.remove(tag)\n",
" return tag, tags\n",
" return None, tags\n",
"\n",
"#each of these variables is a list. Each element of the list represents one instance\n",
"#in text_descriptions, a single element is a list of strings, where each string is a tag associated with the instance.\n",
"#in scores, a single element is the score associated with an instance\n",
"text_descriptions = []\n",
"artists = []\n",
"for input_json in INPUT_JSONS:\n",
" sub_text_descriptions, sub_artists = load_data(input_json)\n",
" text_descriptions.extend(sub_text_descriptions)\n",
" artists.extend(sub_artists)\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "91c66b57",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Artist Count Before Filtering: 57134\n",
"Artist Count After Filtering: 698\n"
]
}
],
"source": [
"# Count the occurrences of each artist\n",
"artist_count = Counter(artists)\n",
"\n",
"# Filter the data to keep only artists with 100 or more occurrences\n",
"min_occurrences = 100\n",
"filtered_text_descriptions = []\n",
"filtered_artists = []\n",
"\n",
"for tags, artist in zip(text_descriptions, artists):\n",
" if artist_count[artist] >= min_occurrences:\n",
" filtered_text_descriptions.append(tags)\n",
" filtered_artists.append(artist)\n",
"\n",
"# Print the result\n",
"print(f\"Artist Count Before Filtering: {len(set(artists))}\")\n",
"print(f\"Artist Count After Filtering: {len(set(filtered_artists))}\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "acf35591",
"metadata": {},
"outputs": [],
"source": [
"from collections import defaultdict\n",
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
"from sklearn.metrics.pairwise import cosine_similarity\n",
"\n",
"\n",
"# Combine the tags of all images for each artist\n",
"artist_tags = defaultdict(list)\n",
"for tags, artist in zip(filtered_text_descriptions, filtered_artists):\n",
" artist_tags[artist].extend(tags)\n",
"\n",
"# Compute the TF-IDF representation for each artist\n",
"vectorizer = TfidfVectorizer(token_pattern=r'[^,]+')\n",
"X_artist = vectorizer.fit_transform([','.join(tags) for tags in artist_tags.values()])\n",
"artist_names = list(artist_tags.keys())"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a232e088",
"metadata": {},
"outputs": [],
"source": [
"# Given a new image with a tag list (excluding the artist name)\n",
"new_image_tags = []\n",
"new_tags_string = \"airplane\"\n",
"new_image_tags.extend(tag.strip() for tag in new_tags_string.split(\",\"))\n",
"\n",
"unseen_tags = set(new_image_tags) - set(vectorizer.vocabulary_.keys())\n",
"print(f'Unseen Tags:{unseen_tags}')\n",
"\n",
"# Compute the TF-IDF representation for the new image\n",
"X_new_image = vectorizer.transform([','.join(new_image_tags)])\n",
"\n",
"# Compute the cosine similarity between the new image and each artist\n",
"similarities = cosine_similarity(X_new_image, X_artist)[0]\n",
"\n",
"# Rank the artists by their similarity scores and select the top 10\n",
"top_n = 20\n",
"\n",
"# Top artists\n",
"top_artist_indices = np.argsort(similarities)[-top_n:][::-1]\n",
"top_artists = [(artist_names[i], similarities[i]) for i in top_artist_indices]\n",
"\n",
"# Bottom artists\n",
"bottom_artist_indices = np.argsort(similarities)[:top_n]\n",
"bottom_artists = [(artist_names[i], similarities[i]) for i in bottom_artist_indices]\n",
"\n",
"# Get the artist names from the top_artists and bottom_artists lists\n",
"top_artist_names = [artist for artist, _ in top_artists]\n",
"bottom_artist_names = [artist for artist, _ in bottom_artists]\n",
"\n",
"# Print the top 10 artists with rank numbers and similarity scores\n",
"print(\"Top 10 artists:\")\n",
"for rank, (artist, score) in enumerate(top_artists, start=1):\n",
" print(f\"{rank}. {artist} - similarity score: {score:.4f}\")\n",
"\n",
"# Print the top 10 artists as a comma-separated list\n",
"print(\"\\nTop 10 artists:\", \", \".join(str(artist) for artist in top_artist_names))\n",
"\n",
"# Print the bottom 10 artists with rank numbers and similarity scores\n",
"print(\"\\nBottom 10 artists:\")\n",
"for rank, (artist, score) in enumerate(bottom_artists, start=1):\n",
" print(f\"{rank}. {artist} - similarity score: {score:.4f}\")\n",
"\n",
"# Print the bottom 10 artists as a comma-separated list\n",
"print(\"\\nBottom 10 artists:\", \", \".join(str(artist) for artist in bottom_artist_names))\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8dbb05e8",
"metadata": {},
"outputs": [],
"source": [
"\n"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "9730cb16",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"\n",
"def calculate_and_save_top_artists(tags, vectorizer, X_artist, artist_names, top_n):\n",
" for tag in tags:\n",
" new_image_tags = [tag.strip() for tag in tag.split(\",\")]\n",
"\n",
" # Compute the TF-IDF representation for the new image\n",
" X_new_image = vectorizer.transform([','.join(new_image_tags)])\n",
"\n",
" # Compute the cosine similarity between the new image and each artist\n",
" similarities = cosine_similarity(X_new_image, X_artist)[0]\n",
"\n",
" # Rank the artists by their similarity scores and select the top\n",
" top_artist_indices = np.argsort(similarities)[-top_n:][::-1]\n",
" top_artists = [(artist_names[i], similarities[i]) for i in top_artist_indices]\n",
"\n",
" # Create dataframes for artists and similarities\n",
" artist_df = pd.DataFrame({tag: [artist for artist, _ in top_artists]}).T\n",
" similarity_df = pd.DataFrame({tag: [f\"{artist}({round(similarity, 3)})\" for artist, similarity in top_artists]}).T\n",
"\n",
" # Append the data to csv files\n",
" artist_df.to_csv('top_artists.csv', mode='a', header=False)\n",
" similarity_df.to_csv('top_artists_similarity.csv', mode='a', header=False)\n",
"\n",
" \n",
"df = pd.read_csv('all_tags.csv')\n",
"unique_sorted_tags = df.iloc[:, 0].tolist()\n",
"# Use the function for all keys in the vocabulary\n",
"calculate_and_save_top_artists(unique_sorted_tags, vectorizer, X_artist, artist_names, 20)\n"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "d38f92b2",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Skipping tag ':3' due to invalid characters in the name.\n",
"Skipping tag ':<' due to invalid characters in the name.\n",
"Skipping tag ':d' due to invalid characters in the name.\n",
"Skipping tag ':o' due to invalid characters in the name.\n",
"Skipping tag '<3' due to invalid characters in the name.\n",
"Skipping tag '<3 censor' due to invalid characters in the name.\n",
"Skipping tag '<3 eyes' due to invalid characters in the name.\n",
"Skipping tag '<3 pupils' due to invalid characters in the name.\n",
"Skipping tag '?!' due to invalid characters in the name.\n",
"Skipping tag 'american dragon: jake long' due to invalid characters in the name.\n",
"Skipping tag 'dust: an elysian tail' due to invalid characters in the name.\n",
"Skipping tag 'five nights at freddy's: security breach' due to invalid characters in the name.\n",
"Skipping tag 'mao mao: heroes of pure heart' due to invalid characters in the name.\n",
"Skipping tag 'spirit: stallion of the cimarron' due to invalid characters in the name.\n"
]
}
],
"source": [
"import pandas as pd\n",
"import os\n",
"\n",
"# Load the csv file\n",
"df = pd.read_csv('top_artists.csv')\n",
"\n",
"# Directory to store the txt files\n",
"output_dir = 'e6ta'\n",
"os.makedirs(output_dir, exist_ok=True) # Make sure the directory exists\n",
"\n",
"# Characters that are not allowed in filenames\n",
"invalid_chars = ['/', '\\\\', ':', '*', '?', '\"', '<', '>', '|']\n",
"\n",
"# Loop through the DataFrame rows\n",
"for index, row in df.iterrows():\n",
" # Get the name for the file (replace spaces with '_')\n",
" filename = row[0].replace(' ', '_') + '.txt'\n",
" \n",
" # Check if the filename contains any invalid characters\n",
" if any(char in filename for char in invalid_chars):\n",
" print(f\"Skipping tag '{row[0]}' due to invalid characters in the name.\")\n",
" continue\n",
"\n",
" # Get the first 10 tags, ignore any that are just whitespace\n",
" tags = [str(tag).strip() for tag in row[1:11] if str(tag).strip()]\n",
"\n",
" # Create the txt file and write the tags\n",
" with open(os.path.join(output_dir, filename), 'w') as f:\n",
" f.write('\\n'.join(tags))\n",
" f.write('\\n') # Add a newline at the end of the file\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "879f5463",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.9"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
|