alxry92 commited on
Commit
ef88fc2
·
verified ·
1 Parent(s): febcd9c

Upload 2 files

Browse files
1_Data_Creation_(2) (3).ipynb ADDED
@@ -0,0 +1,1203 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cells": [
3
+ {
4
+ "cell_type": "markdown",
5
+ "metadata": {
6
+ "id": "4ba6aba8"
7
+ },
8
+ "source": [
9
+ "# 🤖 **Data Collection, Creation, Storage, and Processing**\n"
10
+ ]
11
+ },
12
+ {
13
+ "cell_type": "markdown",
14
+ "metadata": {
15
+ "id": "jpASMyIQMaAq"
16
+ },
17
+ "source": [
18
+ "## **1.** 📦 Install required packages"
19
+ ]
20
+ },
21
+ {
22
+ "cell_type": "code",
23
+ "execution_count": 25,
24
+ "metadata": {
25
+ "colab": {
26
+ "base_uri": "https://localhost:8080/"
27
+ },
28
+ "id": "f48c8f8c",
29
+ "outputId": "2c9eb68f-0716-425f-9eea-12094df9d3ee"
30
+ },
31
+ "outputs": [
32
+ {
33
+ "output_type": "stream",
34
+ "name": "stdout",
35
+ "text": [
36
+ "Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.12/dist-packages (4.13.5)\n",
37
+ "Requirement already satisfied: pandas in /usr/local/lib/python3.12/dist-packages (2.2.2)\n",
38
+ "Requirement already satisfied: matplotlib in /usr/local/lib/python3.12/dist-packages (3.10.0)\n",
39
+ "Requirement already satisfied: seaborn in /usr/local/lib/python3.12/dist-packages (0.13.2)\n",
40
+ "Requirement already satisfied: numpy in /usr/local/lib/python3.12/dist-packages (2.0.2)\n",
41
+ "Requirement already satisfied: textblob in /usr/local/lib/python3.12/dist-packages (0.19.0)\n",
42
+ "Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.12/dist-packages (from beautifulsoup4) (2.8.3)\n",
43
+ "Requirement already satisfied: typing-extensions>=4.0.0 in /usr/local/lib/python3.12/dist-packages (from beautifulsoup4) (4.15.0)\n",
44
+ "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.12/dist-packages (from pandas) (2.9.0.post0)\n",
45
+ "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.12/dist-packages (from pandas) (2025.2)\n",
46
+ "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.12/dist-packages (from pandas) (2025.3)\n",
47
+ "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (1.3.3)\n",
48
+ "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (0.12.1)\n",
49
+ "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (4.61.1)\n",
50
+ "Requirement already satisfied: kiwisolver>=1.3.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (1.4.9)\n",
51
+ "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (26.0)\n",
52
+ "Requirement already satisfied: pillow>=8 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (11.3.0)\n",
53
+ "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.12/dist-packages (from matplotlib) (3.3.2)\n",
54
+ "Requirement already satisfied: nltk>=3.9 in /usr/local/lib/python3.12/dist-packages (from textblob) (3.9.1)\n",
55
+ "Requirement already satisfied: click in /usr/local/lib/python3.12/dist-packages (from nltk>=3.9->textblob) (8.3.1)\n",
56
+ "Requirement already satisfied: joblib in /usr/local/lib/python3.12/dist-packages (from nltk>=3.9->textblob) (1.5.3)\n",
57
+ "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.12/dist-packages (from nltk>=3.9->textblob) (2025.11.3)\n",
58
+ "Requirement already satisfied: tqdm in /usr/local/lib/python3.12/dist-packages (from nltk>=3.9->textblob) (4.67.3)\n",
59
+ "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.12/dist-packages (from python-dateutil>=2.8.2->pandas) (1.17.0)\n"
60
+ ]
61
+ }
62
+ ],
63
+ "source": [
64
+ "!pip install beautifulsoup4 pandas matplotlib seaborn numpy textblob"
65
+ ]
66
+ },
67
+ {
68
+ "cell_type": "markdown",
69
+ "metadata": {
70
+ "id": "lquNYCbfL9IM"
71
+ },
72
+ "source": [
73
+ "## **2.** ⛏ Web-scrape all book titles, prices, and ratings from books.toscrape.com"
74
+ ]
75
+ },
76
+ {
77
+ "cell_type": "markdown",
78
+ "metadata": {
79
+ "id": "0IWuNpxxYDJF"
80
+ },
81
+ "source": [
82
+ "### *a. Initial setup*\n",
83
+ "Define the base url of the website you will scrape as well as how and what you will scrape"
84
+ ]
85
+ },
86
+ {
87
+ "cell_type": "code",
88
+ "execution_count": 26,
89
+ "metadata": {
90
+ "id": "91d52125"
91
+ },
92
+ "outputs": [],
93
+ "source": [
94
+ "import requests\n",
95
+ "from bs4 import BeautifulSoup\n",
96
+ "import pandas as pd\n",
97
+ "import time\n",
98
+ "\n",
99
+ "base_url = \"https://books.toscrape.com/catalogue/page-{}.html\"\n",
100
+ "headers = {\"User-Agent\": \"Mozilla/5.0\"}\n",
101
+ "\n",
102
+ "titles, prices, ratings = [], [], []"
103
+ ]
104
+ },
105
+ {
106
+ "cell_type": "markdown",
107
+ "metadata": {
108
+ "id": "oCdTsin2Yfp3"
109
+ },
110
+ "source": [
111
+ "### *b. Fill titles, prices, and ratings from the web pages*"
112
+ ]
113
+ },
114
+ {
115
+ "cell_type": "code",
116
+ "execution_count": 23,
117
+ "metadata": {
118
+ "id": "xqO5Y3dnYhxt"
119
+ },
120
+ "outputs": [],
121
+ "source": [
122
+ "# Loop through all 50 pages\n",
123
+ "for page in range(1, 51):\n",
124
+ " url = base_url.format(page)\n",
125
+ " response = requests.get(url, headers=headers)\n",
126
+ " soup = BeautifulSoup(response.content, \"html.parser\")\n",
127
+ " books = soup.find_all(\"article\", class_=\"product_pod\")\n",
128
+ "\n",
129
+ " for book in books:\n",
130
+ " titles.append(book.h3.a[\"title\"])\n",
131
+ " prices.append(float(book.find(\"p\", class_=\"price_color\").text[1:]))\n",
132
+ " ratings.append(book.p.get(\"class\")[1])\n",
133
+ "\n",
134
+ " time.sleep(0.5) # polite scraping delay"
135
+ ]
136
+ },
137
+ {
138
+ "cell_type": "markdown",
139
+ "metadata": {
140
+ "id": "T0TOeRC4Yrnn"
141
+ },
142
+ "source": [
143
+ "### *c. ✋🏻🛑⛔️ Create a dataframe df_books that contains the now complete \"title\", \"price\", and \"rating\" objects*"
144
+ ]
145
+ },
146
+ {
147
+ "cell_type": "code",
148
+ "execution_count": 24,
149
+ "metadata": {
150
+ "id": "l5FkkNhUYTHh"
151
+ },
152
+ "outputs": [],
153
+ "source": [
154
+ "import pandas as pd\n",
155
+ "df_books = pd.DataFrame({\n",
156
+ " \"title\": titles,\n",
157
+ " \"price\": prices,\n",
158
+ " \"rating\": ratings\n",
159
+ "})"
160
+ ]
161
+ },
162
+ {
163
+ "cell_type": "markdown",
164
+ "metadata": {
165
+ "id": "duI5dv3CZYvF"
166
+ },
167
+ "source": [
168
+ "### *d. Save web-scraped dataframe either as a CSV or Excel file*"
169
+ ]
170
+ },
171
+ {
172
+ "cell_type": "code",
173
+ "execution_count": 27,
174
+ "metadata": {
175
+ "id": "lC1U_YHtZifh"
176
+ },
177
+ "outputs": [],
178
+ "source": [
179
+ "# 💾 Save to CSV\n",
180
+ "df_books.to_csv(\"books_data.csv\", index=False)\n",
181
+ "\n",
182
+ "# 💾 Or save to Excel\n",
183
+ "# df_books.to_excel(\"books_data.xlsx\", index=False)"
184
+ ]
185
+ },
186
+ {
187
+ "cell_type": "markdown",
188
+ "metadata": {
189
+ "id": "qMjRKMBQZlJi"
190
+ },
191
+ "source": [
192
+ "### *e. ✋🏻🛑⛔️ View first fiew lines*"
193
+ ]
194
+ },
195
+ {
196
+ "cell_type": "code",
197
+ "execution_count": 28,
198
+ "metadata": {
199
+ "colab": {
200
+ "base_uri": "https://localhost:8080/",
201
+ "height": 206
202
+ },
203
+ "id": "O_wIvTxYZqCK",
204
+ "outputId": "86d3f20b-fa44-4fc1-b6aa-306f15780c30"
205
+ },
206
+ "outputs": [
207
+ {
208
+ "output_type": "execute_result",
209
+ "data": {
210
+ "text/plain": [
211
+ " title price rating\n",
212
+ "0 A Light in the Attic 51.77 Three\n",
213
+ "1 Tipping the Velvet 53.74 One\n",
214
+ "2 Soumission 50.10 One\n",
215
+ "3 Sharp Objects 47.82 Four\n",
216
+ "4 Sapiens: A Brief History of Humankind 54.23 Five"
217
+ ],
218
+ "text/html": [
219
+ "\n",
220
+ " <div id=\"df-b206477a-b8f4-448c-be41-650117cff31c\" class=\"colab-df-container\">\n",
221
+ " <div>\n",
222
+ "<style scoped>\n",
223
+ " .dataframe tbody tr th:only-of-type {\n",
224
+ " vertical-align: middle;\n",
225
+ " }\n",
226
+ "\n",
227
+ " .dataframe tbody tr th {\n",
228
+ " vertical-align: top;\n",
229
+ " }\n",
230
+ "\n",
231
+ " .dataframe thead th {\n",
232
+ " text-align: right;\n",
233
+ " }\n",
234
+ "</style>\n",
235
+ "<table border=\"1\" class=\"dataframe\">\n",
236
+ " <thead>\n",
237
+ " <tr style=\"text-align: right;\">\n",
238
+ " <th></th>\n",
239
+ " <th>title</th>\n",
240
+ " <th>price</th>\n",
241
+ " <th>rating</th>\n",
242
+ " </tr>\n",
243
+ " </thead>\n",
244
+ " <tbody>\n",
245
+ " <tr>\n",
246
+ " <th>0</th>\n",
247
+ " <td>A Light in the Attic</td>\n",
248
+ " <td>51.77</td>\n",
249
+ " <td>Three</td>\n",
250
+ " </tr>\n",
251
+ " <tr>\n",
252
+ " <th>1</th>\n",
253
+ " <td>Tipping the Velvet</td>\n",
254
+ " <td>53.74</td>\n",
255
+ " <td>One</td>\n",
256
+ " </tr>\n",
257
+ " <tr>\n",
258
+ " <th>2</th>\n",
259
+ " <td>Soumission</td>\n",
260
+ " <td>50.10</td>\n",
261
+ " <td>One</td>\n",
262
+ " </tr>\n",
263
+ " <tr>\n",
264
+ " <th>3</th>\n",
265
+ " <td>Sharp Objects</td>\n",
266
+ " <td>47.82</td>\n",
267
+ " <td>Four</td>\n",
268
+ " </tr>\n",
269
+ " <tr>\n",
270
+ " <th>4</th>\n",
271
+ " <td>Sapiens: A Brief History of Humankind</td>\n",
272
+ " <td>54.23</td>\n",
273
+ " <td>Five</td>\n",
274
+ " </tr>\n",
275
+ " </tbody>\n",
276
+ "</table>\n",
277
+ "</div>\n",
278
+ " <div class=\"colab-df-buttons\">\n",
279
+ "\n",
280
+ " <div class=\"colab-df-container\">\n",
281
+ " <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-b206477a-b8f4-448c-be41-650117cff31c')\"\n",
282
+ " title=\"Convert this dataframe to an interactive table.\"\n",
283
+ " style=\"display:none;\">\n",
284
+ "\n",
285
+ " <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
286
+ " <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
287
+ " </svg>\n",
288
+ " </button>\n",
289
+ "\n",
290
+ " <style>\n",
291
+ " .colab-df-container {\n",
292
+ " display:flex;\n",
293
+ " gap: 12px;\n",
294
+ " }\n",
295
+ "\n",
296
+ " .colab-df-convert {\n",
297
+ " background-color: #E8F0FE;\n",
298
+ " border: none;\n",
299
+ " border-radius: 50%;\n",
300
+ " cursor: pointer;\n",
301
+ " display: none;\n",
302
+ " fill: #1967D2;\n",
303
+ " height: 32px;\n",
304
+ " padding: 0 0 0 0;\n",
305
+ " width: 32px;\n",
306
+ " }\n",
307
+ "\n",
308
+ " .colab-df-convert:hover {\n",
309
+ " background-color: #E2EBFA;\n",
310
+ " box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
311
+ " fill: #174EA6;\n",
312
+ " }\n",
313
+ "\n",
314
+ " .colab-df-buttons div {\n",
315
+ " margin-bottom: 4px;\n",
316
+ " }\n",
317
+ "\n",
318
+ " [theme=dark] .colab-df-convert {\n",
319
+ " background-color: #3B4455;\n",
320
+ " fill: #D2E3FC;\n",
321
+ " }\n",
322
+ "\n",
323
+ " [theme=dark] .colab-df-convert:hover {\n",
324
+ " background-color: #434B5C;\n",
325
+ " box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
326
+ " filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
327
+ " fill: #FFFFFF;\n",
328
+ " }\n",
329
+ " </style>\n",
330
+ "\n",
331
+ " <script>\n",
332
+ " const buttonEl =\n",
333
+ " document.querySelector('#df-b206477a-b8f4-448c-be41-650117cff31c button.colab-df-convert');\n",
334
+ " buttonEl.style.display =\n",
335
+ " google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
336
+ "\n",
337
+ " async function convertToInteractive(key) {\n",
338
+ " const element = document.querySelector('#df-b206477a-b8f4-448c-be41-650117cff31c');\n",
339
+ " const dataTable =\n",
340
+ " await google.colab.kernel.invokeFunction('convertToInteractive',\n",
341
+ " [key], {});\n",
342
+ " if (!dataTable) return;\n",
343
+ "\n",
344
+ " const docLinkHtml = 'Like what you see? Visit the ' +\n",
345
+ " '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
346
+ " + ' to learn more about interactive tables.';\n",
347
+ " element.innerHTML = '';\n",
348
+ " dataTable['output_type'] = 'display_data';\n",
349
+ " await google.colab.output.renderOutput(dataTable, element);\n",
350
+ " const docLink = document.createElement('div');\n",
351
+ " docLink.innerHTML = docLinkHtml;\n",
352
+ " element.appendChild(docLink);\n",
353
+ " }\n",
354
+ " </script>\n",
355
+ " </div>\n",
356
+ "\n",
357
+ "\n",
358
+ " </div>\n",
359
+ " </div>\n"
360
+ ],
361
+ "application/vnd.google.colaboratory.intrinsic+json": {
362
+ "type": "dataframe",
363
+ "variable_name": "df_books",
364
+ "summary": "{\n \"name\": \"df_books\",\n \"rows\": 1500,\n \"fields\": [\n {\n \"column\": \"title\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 999,\n \"samples\": [\n \"The Grownup\",\n \"Persepolis: The Story of a Childhood (Persepolis #1-2)\",\n \"Ayumi's Violin\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"price\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 14.477392573574008,\n \"min\": 10.0,\n \"max\": 59.99,\n \"num_unique_values\": 903,\n \"samples\": [\n 19.73,\n 55.65,\n 46.31\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"rating\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 5,\n \"samples\": [\n \"One\",\n \"Two\",\n \"Four\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}"
365
+ }
366
+ },
367
+ "metadata": {},
368
+ "execution_count": 28
369
+ }
370
+ ],
371
+ "source": [
372
+ "df_books.head()"
373
+ ]
374
+ },
375
+ {
376
+ "cell_type": "markdown",
377
+ "metadata": {
378
+ "id": "p-1Pr2szaqLk"
379
+ },
380
+ "source": [
381
+ "## **3.** 🧩 Create a meaningful connection between real & synthetic datasets"
382
+ ]
383
+ },
384
+ {
385
+ "cell_type": "markdown",
386
+ "metadata": {
387
+ "id": "SIaJUGIpaH4V"
388
+ },
389
+ "source": [
390
+ "### *a. Initial setup*"
391
+ ]
392
+ },
393
+ {
394
+ "cell_type": "code",
395
+ "execution_count": 29,
396
+ "metadata": {
397
+ "id": "-gPXGcRPuV_9"
398
+ },
399
+ "outputs": [],
400
+ "source": [
401
+ "import numpy as np\n",
402
+ "import random\n",
403
+ "from datetime import datetime\n",
404
+ "import warnings\n",
405
+ "\n",
406
+ "warnings.filterwarnings(\"ignore\")\n",
407
+ "random.seed(2025)\n",
408
+ "np.random.seed(2025)"
409
+ ]
410
+ },
411
+ {
412
+ "cell_type": "markdown",
413
+ "metadata": {
414
+ "id": "pY4yCoIuaQqp"
415
+ },
416
+ "source": [
417
+ "### *b. Generate popularity scores based on rating (with some randomness) with a generate_popularity_score function*"
418
+ ]
419
+ },
420
+ {
421
+ "cell_type": "code",
422
+ "execution_count": 30,
423
+ "metadata": {
424
+ "id": "mnd5hdAbaNjz"
425
+ },
426
+ "outputs": [],
427
+ "source": [
428
+ "def generate_popularity_score(rating):\n",
429
+ " base = {\"One\": 2, \"Two\": 3, \"Three\": 3, \"Four\": 4, \"Five\": 4}.get(rating, 3)\n",
430
+ " trend_factor = random.choices([-1, 0, 1], weights=[1, 3, 2])[0]\n",
431
+ " return int(np.clip(base + trend_factor, 1, 5))"
432
+ ]
433
+ },
434
+ {
435
+ "cell_type": "markdown",
436
+ "metadata": {
437
+ "id": "n4-TaNTFgPak"
438
+ },
439
+ "source": [
440
+ "### *c. ✋🏻🛑⛔️ Run the function to create a \"popularity_score\" column from \"rating\"*"
441
+ ]
442
+ },
443
+ {
444
+ "cell_type": "code",
445
+ "execution_count": 31,
446
+ "metadata": {
447
+ "id": "V-G3OCUCgR07"
448
+ },
449
+ "outputs": [],
450
+ "source": [
451
+ "df_books[\"popularity_score\"] = df_books[\"rating\"].apply(generate_popularity_score)"
452
+ ]
453
+ },
454
+ {
455
+ "cell_type": "markdown",
456
+ "metadata": {
457
+ "id": "HnngRNTgacYt"
458
+ },
459
+ "source": [
460
+ "### *d. Decide on the sentiment_label based on the popularity score with a get_sentiment function*"
461
+ ]
462
+ },
463
+ {
464
+ "cell_type": "code",
465
+ "execution_count": 32,
466
+ "metadata": {
467
+ "id": "kUtWmr8maZLZ"
468
+ },
469
+ "outputs": [],
470
+ "source": [
471
+ "def get_sentiment(popularity_score):\n",
472
+ " if popularity_score <= 2:\n",
473
+ " return \"negative\"\n",
474
+ " elif popularity_score == 3:\n",
475
+ " return \"neutral\"\n",
476
+ " else:\n",
477
+ " return \"positive\""
478
+ ]
479
+ },
480
+ {
481
+ "cell_type": "markdown",
482
+ "metadata": {
483
+ "id": "HF9F9HIzgT7Z"
484
+ },
485
+ "source": [
486
+ "### *e. ✋🏻🛑⛔️ Run the function to create a \"sentiment_label\" column from \"popularity_score\"*"
487
+ ]
488
+ },
489
+ {
490
+ "cell_type": "code",
491
+ "execution_count": 33,
492
+ "metadata": {
493
+ "id": "tafQj8_7gYCG"
494
+ },
495
+ "outputs": [],
496
+ "source": [
497
+ "df_books[\"sentiment_label\"] = df_books[\"popularity_score\"].apply(get_sentiment)"
498
+ ]
499
+ },
500
+ {
501
+ "cell_type": "markdown",
502
+ "metadata": {
503
+ "id": "T8AdKkmASq9a"
504
+ },
505
+ "source": [
506
+ "## **4.** 📈 Generate synthetic book sales data of 18 months"
507
+ ]
508
+ },
509
+ {
510
+ "cell_type": "markdown",
511
+ "metadata": {
512
+ "id": "OhXbdGD5fH0c"
513
+ },
514
+ "source": [
515
+ "### *a. Create a generate_sales_profit function that would generate sales patterns based on sentiment_label (with some randomness)*"
516
+ ]
517
+ },
518
+ {
519
+ "cell_type": "code",
520
+ "execution_count": 34,
521
+ "metadata": {
522
+ "id": "qkVhYPXGbgEn"
523
+ },
524
+ "outputs": [],
525
+ "source": [
526
+ "def generate_sales_profile(sentiment):\n",
527
+ " months = pd.date_range(end=datetime.today(), periods=18, freq=\"M\")\n",
528
+ "\n",
529
+ " if sentiment == \"positive\":\n",
530
+ " base = random.randint(200, 300)\n",
531
+ " trend = np.linspace(base, base + random.randint(20, 60), len(months))\n",
532
+ " elif sentiment == \"negative\":\n",
533
+ " base = random.randint(20, 80)\n",
534
+ " trend = np.linspace(base, base - random.randint(10, 30), len(months))\n",
535
+ " else: # neutral\n",
536
+ " base = random.randint(80, 160)\n",
537
+ " trend = np.full(len(months), base + random.randint(-10, 10))\n",
538
+ "\n",
539
+ " seasonality = 10 * np.sin(np.linspace(0, 3 * np.pi, len(months)))\n",
540
+ " noise = np.random.normal(0, 5, len(months))\n",
541
+ " monthly_sales = np.clip(trend + seasonality + noise, a_min=0, a_max=None).astype(int)\n",
542
+ "\n",
543
+ " return list(zip(months.strftime(\"%Y-%m\"), monthly_sales))"
544
+ ]
545
+ },
546
+ {
547
+ "cell_type": "markdown",
548
+ "metadata": {
549
+ "id": "L2ak1HlcgoTe"
550
+ },
551
+ "source": [
552
+ "### *b. Run the function as part of building sales_data*"
553
+ ]
554
+ },
555
+ {
556
+ "cell_type": "code",
557
+ "execution_count": 35,
558
+ "metadata": {
559
+ "id": "SlJ24AUafoDB"
560
+ },
561
+ "outputs": [],
562
+ "source": [
563
+ "sales_data = []\n",
564
+ "for _, row in df_books.iterrows():\n",
565
+ " records = generate_sales_profile(row[\"sentiment_label\"])\n",
566
+ " for month, units in records:\n",
567
+ " sales_data.append({\n",
568
+ " \"title\": row[\"title\"],\n",
569
+ " \"month\": month,\n",
570
+ " \"units_sold\": units,\n",
571
+ " \"sentiment_label\": row[\"sentiment_label\"]\n",
572
+ " })"
573
+ ]
574
+ },
575
+ {
576
+ "cell_type": "markdown",
577
+ "metadata": {
578
+ "id": "4IXZKcCSgxnq"
579
+ },
580
+ "source": [
581
+ "### *c. ✋🏻🛑⛔️ Create a df_sales DataFrame from sales_data*"
582
+ ]
583
+ },
584
+ {
585
+ "cell_type": "code",
586
+ "execution_count": 37,
587
+ "metadata": {
588
+ "id": "wcN6gtiZg-ws"
589
+ },
590
+ "outputs": [],
591
+ "source": [
592
+ "df_sales = pd.DataFrame(sales_data)"
593
+ ]
594
+ },
595
+ {
596
+ "cell_type": "markdown",
597
+ "metadata": {
598
+ "id": "EhIjz9WohAmZ"
599
+ },
600
+ "source": [
601
+ "### *d. Save df_sales as synthetic_sales_data.csv & view first few lines*"
602
+ ]
603
+ },
604
+ {
605
+ "cell_type": "code",
606
+ "execution_count": 38,
607
+ "metadata": {
608
+ "colab": {
609
+ "base_uri": "https://localhost:8080/"
610
+ },
611
+ "id": "MzbZvLcAhGaH",
612
+ "outputId": "312074f9-b3dc-440e-a0e0-e48642fbd8ca"
613
+ },
614
+ "outputs": [
615
+ {
616
+ "output_type": "stream",
617
+ "name": "stdout",
618
+ "text": [
619
+ " title month units_sold sentiment_label\n",
620
+ "0 A Light in the Attic 2024-09 79 neutral\n",
621
+ "1 A Light in the Attic 2024-10 88 neutral\n",
622
+ "2 A Light in the Attic 2024-11 81 neutral\n",
623
+ "3 A Light in the Attic 2024-12 86 neutral\n",
624
+ "4 A Light in the Attic 2025-01 87 neutral\n"
625
+ ]
626
+ }
627
+ ],
628
+ "source": [
629
+ "df_sales.to_csv(\"synthetic_sales_data.csv\", index=False)\n",
630
+ "\n",
631
+ "print(df_sales.head())"
632
+ ]
633
+ },
634
+ {
635
+ "cell_type": "markdown",
636
+ "metadata": {
637
+ "id": "7g9gqBgQMtJn"
638
+ },
639
+ "source": [
640
+ "## **5.** 🎯 Generate synthetic customer reviews"
641
+ ]
642
+ },
643
+ {
644
+ "cell_type": "markdown",
645
+ "metadata": {
646
+ "id": "Gi4y9M9KuDWx"
647
+ },
648
+ "source": [
649
+ "### *a. ✋🏻🛑⛔️ Ask ChatGPT to create a list of 50 distinct generic book review texts for the sentiment labels \"positive\", \"neutral\", and \"negative\" called synthetic_reviews_by_sentiment*"
650
+ ]
651
+ },
652
+ {
653
+ "cell_type": "code",
654
+ "execution_count": 40,
655
+ "metadata": {
656
+ "id": "b3cd2a50"
657
+ },
658
+ "outputs": [],
659
+ "source": [
660
+ "synthetic_reviews_by_sentiment = {\n",
661
+ " \"positive\": [\n",
662
+ " \"A compelling and heartwarming read that stayed with me long after I finished.\",\n",
663
+ " \"Brilliantly written! The characters were unforgettable and the plot was engaging.\",\n",
664
+ " \"One of the best books I've read this year — inspiring and emotionally rich.\",\n",
665
+ " \"A masterpiece of storytelling. Every page felt purposeful and alive.\",\n",
666
+ " \"I couldn't put it down. The pacing was perfect and the ending left me breathless.\",\n",
667
+ " \"An absolute gem. The author has a gift for making you care deeply about every character.\",\n",
668
+ " \"Beautifully crafted. The prose is lyrical and the themes resonate deeply.\",\n",
669
+ " \"A refreshing and original story that exceeded all my expectations.\",\n",
670
+ " \"Utterly captivating from the first page to the last. A must-read.\",\n",
671
+ " \"The world-building was extraordinary — I felt completely immersed.\",\n",
672
+ " \"This book changed the way I see the world. Profound and thought-provoking.\",\n",
673
+ " \"A joy to read. Witty, warm, and full of heart.\",\n",
674
+ " \"I laughed, I cried, and I didn't want it to end. Truly special.\",\n",
675
+ " \"The character development was exceptional — I grew to love them all.\",\n",
676
+ " \"A beautifully told story with depth, nuance, and genuine emotional impact.\",\n",
677
+ " \"One of those rare books that feels like it was written just for you.\",\n",
678
+ " \"Gripping, intelligent, and deeply moving. Highly recommend.\",\n",
679
+ " \"The writing is sharp and confident. A truly impressive debut.\",\n",
680
+ " \"I've recommended this to everyone I know. It's just that good.\",\n",
681
+ " \"An uplifting and unforgettable story that restores your faith in humanity.\",\n",
682
+ " \"Every chapter pulled me deeper in. I read the whole thing in one sitting.\",\n",
683
+ " \"The dialogue felt so real and natural — the characters jumped off the page.\",\n",
684
+ " \"A powerful narrative with a message that will stay with me for years.\",\n",
685
+ " \"Simply wonderful. Funny, touching, and beautifully observed.\",\n",
686
+ " \"A rare find — a book that is both entertaining and genuinely meaningful.\",\n",
687
+ " \"The plot twists were stunning. I never saw them coming but they felt earned.\",\n",
688
+ " \"Rich in detail and atmosphere. The author clearly did their research.\",\n",
689
+ " \"An emotional rollercoaster in the best possible way.\",\n",
690
+ " \"Flawlessly executed. This is exactly what great fiction should feel like.\",\n",
691
+ " \"A brilliant exploration of the human condition, told with grace and insight.\",\n",
692
+ " \"I finished this with a huge smile on my face. Pure reading pleasure.\",\n",
693
+ " \"Warm, wise, and wonderfully written. A new favourite.\",\n",
694
+ " \"The kind of book that reminds you why you love reading.\",\n",
695
+ " \"Exceptional storytelling — complex, compassionate, and completely absorbing.\"\n",
696
+ " ],\n",
697
+ " \"neutral\": [\n",
698
+ " \"An average book — not great, but not bad either.\",\n",
699
+ " \"Some parts really stood out, others felt a bit flat.\",\n",
700
+ " \"It was okay overall. A decent way to pass the time.\",\n",
701
+ " \"Interesting enough premise, but the execution was a bit uneven.\",\n",
702
+ " \"I enjoyed parts of it, though it didn't quite live up to the hype.\",\n",
703
+ " \"A solid read, but nothing I'd rush to recommend.\",\n",
704
+ " \"The story had potential, but it felt like it never fully delivered.\",\n",
705
+ " \"Not bad, not great. I finished it, which counts for something.\",\n",
706
+ " \"Some chapters were genuinely gripping; others dragged on a bit.\",\n",
707
+ " \"A competent book. Well-written in places, forgettable in others.\",\n",
708
+ " \"It kept my interest without ever truly exciting me.\",\n",
709
+ " \"A middle-of-the-road read — pleasant but unmemorable.\",\n",
710
+ " \"The writing was fine. The story was fine. Overall, just fine.\",\n",
711
+ " \"I can see why others loved it, even if it didn't click for me personally.\",\n",
712
+ " \"Has its moments, but also has its lulls. A mixed bag.\",\n",
713
+ " \"Worth reading if you're a fan of the genre, but it won't blow you away.\",\n",
714
+ " \"I liked the concept more than the execution.\",\n",
715
+ " \"A reasonable effort. I don't regret reading it, but I wouldn't reread it.\",\n",
716
+ " \"The first half was stronger than the second.\",\n",
717
+ " \"Enjoyable in a lightweight way. Nothing that will linger long.\",\n",
718
+ " \"Not particularly original, but competently done.\",\n",
719
+ " \"A functional story that does what it sets out to do, nothing more.\",\n",
720
+ " \"The characters were likeable enough, if not especially memorable.\",\n",
721
+ " \"It passed the time pleasantly. No strong feelings either way.\",\n",
722
+ " \"Somewhere between a 3 and a 4 out of 5 — solidly average.\",\n",
723
+ " \"Some good ideas here, but they weren't fully developed.\",\n",
724
+ " \"The plot moved along, though I never felt urgently compelled to keep reading.\",\n",
725
+ " \"A book that plays it safe — comforting but not particularly challenging.\",\n",
726
+ " \"Decent writing, predictable story. A passable read.\",\n",
727
+ " \"I'd neither seek out this author again nor actively avoid them.\",\n",
728
+ " \"It scratched the itch for light reading, without being particularly memorable.\",\n",
729
+ " \"Readable and inoffensive, if not especially inspiring.\",\n",
730
+ " \"The ending felt a little too neat, but the journey was okay.\"\n",
731
+ " ],\n",
732
+ " \"negative\": [\n",
733
+ " \"I struggled to get through this one — it just didn't grab me.\",\n",
734
+ " \"The plot was confusing and the characters felt underdeveloped.\",\n",
735
+ " \"Disappointing. I had high hopes, but they weren't met.\",\n",
736
+ " \"I forced myself to finish it but honestly wish I hadn't bothered.\",\n",
737
+ " \"The pacing was painfully slow and I found myself skipping pages.\",\n",
738
+ " \"The characters were flat and I couldn't connect with any of them.\",\n",
739
+ " \"Overhyped and underwhelming. Not sure what all the fuss is about.\",\n",
740
+ " \"The writing felt clunky and the dialogue was unconvincing.\",\n",
741
+ " \"I lost the plot entirely around chapter five and never recovered.\",\n",
742
+ " \"A promising premise completely wasted by poor execution.\",\n",
743
+ " \"Tedious and repetitive. The story went in circles.\",\n",
744
+ " \"I kept waiting for it to get interesting. It never did.\",\n",
745
+ " \"The ending was a massive letdown after an already slow build.\",\n",
746
+ " \"I found the main character deeply unlikeable and hard to root for.\",\n",
747
+ " \"Far too long for the story it was trying to tell.\",\n",
748
+ " \"The world-building was confusing and the rules kept changing.\",\n",
749
+ " \"I've read better books in the same genre that cost half the price.\",\n",
750
+ " \"Felt like a first draft. Could have used a much stronger edit.\",\n",
751
+ " \"The tone was inconsistent — I couldn't tell if it was trying to be funny or serious.\",\n",
752
+ " \"Not for me at all. I abandoned it at the halfway mark.\",\n",
753
+ " \"The prose was purple and overwrought. Less would have been more.\",\n",
754
+ " \"I found the themes heavy-handed and the message preachy.\",\n",
755
+ " \"Predictable from page one. Nothing surprised or delighted me.\",\n",
756
+ " \"The secondary characters were more interesting than the lead, which was a problem.\",\n",
757
+ " \"A real struggle to finish. I persisted out of stubbornness, not enjoyment.\",\n",
758
+ " \"The plot holes were too large to ignore and broke my immersion completely.\",\n",
759
+ " \"Felt generic and derivative. I've seen this story told much better elsewhere.\",\n",
760
+ " \"The romance subplot felt forced and took away from the main story.\",\n",
761
+ " \"Poorly structured — the story jumped around in a way that felt chaotic.\",\n",
762
+ " \"I didn't care what happened to any of the characters by the end.\",\n",
763
+ " \"The humour fell flat for me throughout. More cringeworthy than funny.\",\n",
764
+ " \"A disappointment from start to finish. I won't be reading the sequel.\",\n",
765
+ " \"Too much telling and not enough showing. A basic craft issue.\"\n",
766
+ " ]\n",
767
+ "}"
768
+ ]
769
+ },
770
+ {
771
+ "cell_type": "markdown",
772
+ "metadata": {
773
+ "id": "fQhfVaDmuULT"
774
+ },
775
+ "source": [
776
+ "### *b. Generate 10 reviews per book using random sampling from the corresponding 50*"
777
+ ]
778
+ },
779
+ {
780
+ "cell_type": "code",
781
+ "execution_count": 41,
782
+ "metadata": {
783
+ "id": "l2SRc3PjuTGM"
784
+ },
785
+ "outputs": [],
786
+ "source": [
787
+ "review_rows = []\n",
788
+ "for _, row in df_books.iterrows():\n",
789
+ " title = row['title']\n",
790
+ " sentiment_label = row['sentiment_label']\n",
791
+ " review_pool = synthetic_reviews_by_sentiment[sentiment_label]\n",
792
+ " sampled_reviews = random.sample(review_pool, 10)\n",
793
+ " for review_text in sampled_reviews:\n",
794
+ " review_rows.append({\n",
795
+ " \"title\": title,\n",
796
+ " \"sentiment_label\": sentiment_label,\n",
797
+ " \"review_text\": review_text,\n",
798
+ " \"rating\": row['rating'],\n",
799
+ " \"popularity_score\": row['popularity_score']\n",
800
+ " })"
801
+ ]
802
+ },
803
+ {
804
+ "cell_type": "markdown",
805
+ "metadata": {
806
+ "id": "bmJMXF-Bukdm"
807
+ },
808
+ "source": [
809
+ "### *c. Create the final dataframe df_reviews & save it as synthetic_book_reviews.csv*"
810
+ ]
811
+ },
812
+ {
813
+ "cell_type": "code",
814
+ "execution_count": 42,
815
+ "metadata": {
816
+ "id": "ZUKUqZsuumsp"
817
+ },
818
+ "outputs": [],
819
+ "source": [
820
+ "df_reviews = pd.DataFrame(review_rows)\n",
821
+ "df_reviews.to_csv(\"synthetic_book_reviews.csv\", index=False)"
822
+ ]
823
+ },
824
+ {
825
+ "cell_type": "markdown",
826
+ "source": [
827
+ "### *c. inputs for R*"
828
+ ],
829
+ "metadata": {
830
+ "id": "_602pYUS3gY5"
831
+ }
832
+ },
833
+ {
834
+ "cell_type": "code",
835
+ "execution_count": 43,
836
+ "metadata": {
837
+ "colab": {
838
+ "base_uri": "https://localhost:8080/"
839
+ },
840
+ "id": "3946e521",
841
+ "outputId": "64c9005b-3916-4d9b-b9fa-a5b90ad256d7"
842
+ },
843
+ "outputs": [
844
+ {
845
+ "output_type": "stream",
846
+ "name": "stdout",
847
+ "text": [
848
+ "✅ Wrote synthetic_title_level_features.csv\n",
849
+ "✅ Wrote synthetic_monthly_revenue_series.csv\n"
850
+ ]
851
+ }
852
+ ],
853
+ "source": [
854
+ "import numpy as np\n",
855
+ "\n",
856
+ "def _safe_num(s):\n",
857
+ " return pd.to_numeric(\n",
858
+ " pd.Series(s).astype(str).str.replace(r\"[^0-9.]\", \"\", regex=True),\n",
859
+ " errors=\"coerce\"\n",
860
+ " )\n",
861
+ "\n",
862
+ "# --- Clean book metadata (price/rating) ---\n",
863
+ "df_books_r = df_books.copy()\n",
864
+ "if \"price\" in df_books_r.columns:\n",
865
+ " df_books_r[\"price\"] = _safe_num(df_books_r[\"price\"])\n",
866
+ "if \"rating\" in df_books_r.columns:\n",
867
+ " df_books_r[\"rating\"] = _safe_num(df_books_r[\"rating\"])\n",
868
+ "\n",
869
+ "df_books_r[\"title\"] = df_books_r[\"title\"].astype(str).str.strip()\n",
870
+ "\n",
871
+ "# --- Clean sales ---\n",
872
+ "df_sales_r = df_sales.copy()\n",
873
+ "df_sales_r[\"title\"] = df_sales_r[\"title\"].astype(str).str.strip()\n",
874
+ "df_sales_r[\"month\"] = pd.to_datetime(df_sales_r[\"month\"], errors=\"coerce\")\n",
875
+ "df_sales_r[\"units_sold\"] = _safe_num(df_sales_r[\"units_sold\"])\n",
876
+ "\n",
877
+ "# --- Clean reviews ---\n",
878
+ "df_reviews_r = df_reviews.copy()\n",
879
+ "df_reviews_r[\"title\"] = df_reviews_r[\"title\"].astype(str).str.strip()\n",
880
+ "df_reviews_r[\"sentiment_label\"] = df_reviews_r[\"sentiment_label\"].astype(str).str.lower().str.strip()\n",
881
+ "if \"rating\" in df_reviews_r.columns:\n",
882
+ " df_reviews_r[\"rating\"] = _safe_num(df_reviews_r[\"rating\"])\n",
883
+ "if \"popularity_score\" in df_reviews_r.columns:\n",
884
+ " df_reviews_r[\"popularity_score\"] = _safe_num(df_reviews_r[\"popularity_score\"])\n",
885
+ "\n",
886
+ "# --- Sentiment shares per title (from reviews) ---\n",
887
+ "sent_counts = (\n",
888
+ " df_reviews_r.groupby([\"title\", \"sentiment_label\"])\n",
889
+ " .size()\n",
890
+ " .unstack(fill_value=0)\n",
891
+ ")\n",
892
+ "for lab in [\"positive\", \"neutral\", \"negative\"]:\n",
893
+ " if lab not in sent_counts.columns:\n",
894
+ " sent_counts[lab] = 0\n",
895
+ "\n",
896
+ "sent_counts[\"total_reviews\"] = sent_counts[[\"positive\", \"neutral\", \"negative\"]].sum(axis=1)\n",
897
+ "den = sent_counts[\"total_reviews\"].replace(0, np.nan)\n",
898
+ "sent_counts[\"share_positive\"] = sent_counts[\"positive\"] / den\n",
899
+ "sent_counts[\"share_neutral\"] = sent_counts[\"neutral\"] / den\n",
900
+ "sent_counts[\"share_negative\"] = sent_counts[\"negative\"] / den\n",
901
+ "sent_counts = sent_counts.reset_index()\n",
902
+ "\n",
903
+ "# --- Sales aggregation per title ---\n",
904
+ "sales_by_title = (\n",
905
+ " df_sales_r.dropna(subset=[\"title\"])\n",
906
+ " .groupby(\"title\", as_index=False)\n",
907
+ " .agg(\n",
908
+ " months_observed=(\"month\", \"nunique\"),\n",
909
+ " avg_units_sold=(\"units_sold\", \"mean\"),\n",
910
+ " total_units_sold=(\"units_sold\", \"sum\"),\n",
911
+ " )\n",
912
+ ")\n",
913
+ "\n",
914
+ "# --- Title-level features (join sales + books + sentiment) ---\n",
915
+ "df_title = (\n",
916
+ " sales_by_title\n",
917
+ " .merge(df_books_r[[\"title\", \"price\", \"rating\"]], on=\"title\", how=\"left\")\n",
918
+ " .merge(sent_counts[[\"title\", \"share_positive\", \"share_neutral\", \"share_negative\", \"total_reviews\"]],\n",
919
+ " on=\"title\", how=\"left\")\n",
920
+ ")\n",
921
+ "\n",
922
+ "df_title[\"avg_revenue\"] = df_title[\"avg_units_sold\"] * df_title[\"price\"]\n",
923
+ "df_title[\"total_revenue\"] = df_title[\"total_units_sold\"] * df_title[\"price\"]\n",
924
+ "\n",
925
+ "df_title.to_csv(\"synthetic_title_level_features.csv\", index=False)\n",
926
+ "print(\"✅ Wrote synthetic_title_level_features.csv\")\n",
927
+ "\n",
928
+ "# --- Monthly revenue series (proxy: units_sold * price) ---\n",
929
+ "monthly_rev = (\n",
930
+ " df_sales_r.merge(df_books_r[[\"title\", \"price\"]], on=\"title\", how=\"left\")\n",
931
+ ")\n",
932
+ "monthly_rev[\"revenue\"] = monthly_rev[\"units_sold\"] * monthly_rev[\"price\"]\n",
933
+ "\n",
934
+ "df_monthly = (\n",
935
+ " monthly_rev.dropna(subset=[\"month\"])\n",
936
+ " .groupby(\"month\", as_index=False)[\"revenue\"]\n",
937
+ " .sum()\n",
938
+ " .rename(columns={\"revenue\": \"total_revenue\"})\n",
939
+ " .sort_values(\"month\")\n",
940
+ ")\n",
941
+ "# if revenue is all NA (e.g., missing price), fallback to units_sold as a teaching proxy\n",
942
+ "if df_monthly[\"total_revenue\"].notna().sum() == 0:\n",
943
+ " df_monthly = (\n",
944
+ " df_sales_r.dropna(subset=[\"month\"])\n",
945
+ " .groupby(\"month\", as_index=False)[\"units_sold\"]\n",
946
+ " .sum()\n",
947
+ " .rename(columns={\"units_sold\": \"total_revenue\"})\n",
948
+ " .sort_values(\"month\")\n",
949
+ " )\n",
950
+ "\n",
951
+ "df_monthly[\"month\"] = pd.to_datetime(df_monthly[\"month\"], errors=\"coerce\").dt.strftime(\"%Y-%m-%d\")\n",
952
+ "df_monthly.to_csv(\"synthetic_monthly_revenue_series.csv\", index=False)\n",
953
+ "print(\"✅ Wrote synthetic_monthly_revenue_series.csv\")\n"
954
+ ]
955
+ },
956
+ {
957
+ "cell_type": "markdown",
958
+ "metadata": {
959
+ "id": "RYvGyVfXuo54"
960
+ },
961
+ "source": [
962
+ "### *d. ✋🏻🛑⛔️ View the first few lines*"
963
+ ]
964
+ },
965
+ {
966
+ "cell_type": "code",
967
+ "execution_count": 44,
968
+ "metadata": {
969
+ "colab": {
970
+ "base_uri": "https://localhost:8080/",
971
+ "height": 206
972
+ },
973
+ "id": "xfE8NMqOurKo",
974
+ "outputId": "87a8b891-5b0a-4581-d1e9-e6c3052f569a"
975
+ },
976
+ "outputs": [
977
+ {
978
+ "output_type": "execute_result",
979
+ "data": {
980
+ "text/plain": [
981
+ " title sentiment_label \\\n",
982
+ "0 A Light in the Attic neutral \n",
983
+ "1 A Light in the Attic neutral \n",
984
+ "2 A Light in the Attic neutral \n",
985
+ "3 A Light in the Attic neutral \n",
986
+ "4 A Light in the Attic neutral \n",
987
+ "\n",
988
+ " review_text rating popularity_score \n",
989
+ "0 A solid read, but nothing I'd rush to recommend. Three 3 \n",
990
+ "1 It kept my interest without ever truly excitin... Three 3 \n",
991
+ "2 Enjoyable in a lightweight way. Nothing that w... Three 3 \n",
992
+ "3 Has its moments, but also has its lulls. A mix... Three 3 \n",
993
+ "4 Worth reading if you're a fan of the genre, bu... Three 3 "
994
+ ],
995
+ "text/html": [
996
+ "\n",
997
+ " <div id=\"df-df099fdf-fd04-4b7a-82e8-3749cda27cda\" class=\"colab-df-container\">\n",
998
+ " <div>\n",
999
+ "<style scoped>\n",
1000
+ " .dataframe tbody tr th:only-of-type {\n",
1001
+ " vertical-align: middle;\n",
1002
+ " }\n",
1003
+ "\n",
1004
+ " .dataframe tbody tr th {\n",
1005
+ " vertical-align: top;\n",
1006
+ " }\n",
1007
+ "\n",
1008
+ " .dataframe thead th {\n",
1009
+ " text-align: right;\n",
1010
+ " }\n",
1011
+ "</style>\n",
1012
+ "<table border=\"1\" class=\"dataframe\">\n",
1013
+ " <thead>\n",
1014
+ " <tr style=\"text-align: right;\">\n",
1015
+ " <th></th>\n",
1016
+ " <th>title</th>\n",
1017
+ " <th>sentiment_label</th>\n",
1018
+ " <th>review_text</th>\n",
1019
+ " <th>rating</th>\n",
1020
+ " <th>popularity_score</th>\n",
1021
+ " </tr>\n",
1022
+ " </thead>\n",
1023
+ " <tbody>\n",
1024
+ " <tr>\n",
1025
+ " <th>0</th>\n",
1026
+ " <td>A Light in the Attic</td>\n",
1027
+ " <td>neutral</td>\n",
1028
+ " <td>A solid read, but nothing I'd rush to recommend.</td>\n",
1029
+ " <td>Three</td>\n",
1030
+ " <td>3</td>\n",
1031
+ " </tr>\n",
1032
+ " <tr>\n",
1033
+ " <th>1</th>\n",
1034
+ " <td>A Light in the Attic</td>\n",
1035
+ " <td>neutral</td>\n",
1036
+ " <td>It kept my interest without ever truly excitin...</td>\n",
1037
+ " <td>Three</td>\n",
1038
+ " <td>3</td>\n",
1039
+ " </tr>\n",
1040
+ " <tr>\n",
1041
+ " <th>2</th>\n",
1042
+ " <td>A Light in the Attic</td>\n",
1043
+ " <td>neutral</td>\n",
1044
+ " <td>Enjoyable in a lightweight way. Nothing that w...</td>\n",
1045
+ " <td>Three</td>\n",
1046
+ " <td>3</td>\n",
1047
+ " </tr>\n",
1048
+ " <tr>\n",
1049
+ " <th>3</th>\n",
1050
+ " <td>A Light in the Attic</td>\n",
1051
+ " <td>neutral</td>\n",
1052
+ " <td>Has its moments, but also has its lulls. A mix...</td>\n",
1053
+ " <td>Three</td>\n",
1054
+ " <td>3</td>\n",
1055
+ " </tr>\n",
1056
+ " <tr>\n",
1057
+ " <th>4</th>\n",
1058
+ " <td>A Light in the Attic</td>\n",
1059
+ " <td>neutral</td>\n",
1060
+ " <td>Worth reading if you're a fan of the genre, bu...</td>\n",
1061
+ " <td>Three</td>\n",
1062
+ " <td>3</td>\n",
1063
+ " </tr>\n",
1064
+ " </tbody>\n",
1065
+ "</table>\n",
1066
+ "</div>\n",
1067
+ " <div class=\"colab-df-buttons\">\n",
1068
+ "\n",
1069
+ " <div class=\"colab-df-container\">\n",
1070
+ " <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-df099fdf-fd04-4b7a-82e8-3749cda27cda')\"\n",
1071
+ " title=\"Convert this dataframe to an interactive table.\"\n",
1072
+ " style=\"display:none;\">\n",
1073
+ "\n",
1074
+ " <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
1075
+ " <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
1076
+ " </svg>\n",
1077
+ " </button>\n",
1078
+ "\n",
1079
+ " <style>\n",
1080
+ " .colab-df-container {\n",
1081
+ " display:flex;\n",
1082
+ " gap: 12px;\n",
1083
+ " }\n",
1084
+ "\n",
1085
+ " .colab-df-convert {\n",
1086
+ " background-color: #E8F0FE;\n",
1087
+ " border: none;\n",
1088
+ " border-radius: 50%;\n",
1089
+ " cursor: pointer;\n",
1090
+ " display: none;\n",
1091
+ " fill: #1967D2;\n",
1092
+ " height: 32px;\n",
1093
+ " padding: 0 0 0 0;\n",
1094
+ " width: 32px;\n",
1095
+ " }\n",
1096
+ "\n",
1097
+ " .colab-df-convert:hover {\n",
1098
+ " background-color: #E2EBFA;\n",
1099
+ " box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
1100
+ " fill: #174EA6;\n",
1101
+ " }\n",
1102
+ "\n",
1103
+ " .colab-df-buttons div {\n",
1104
+ " margin-bottom: 4px;\n",
1105
+ " }\n",
1106
+ "\n",
1107
+ " [theme=dark] .colab-df-convert {\n",
1108
+ " background-color: #3B4455;\n",
1109
+ " fill: #D2E3FC;\n",
1110
+ " }\n",
1111
+ "\n",
1112
+ " [theme=dark] .colab-df-convert:hover {\n",
1113
+ " background-color: #434B5C;\n",
1114
+ " box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
1115
+ " filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
1116
+ " fill: #FFFFFF;\n",
1117
+ " }\n",
1118
+ " </style>\n",
1119
+ "\n",
1120
+ " <script>\n",
1121
+ " const buttonEl =\n",
1122
+ " document.querySelector('#df-df099fdf-fd04-4b7a-82e8-3749cda27cda button.colab-df-convert');\n",
1123
+ " buttonEl.style.display =\n",
1124
+ " google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
1125
+ "\n",
1126
+ " async function convertToInteractive(key) {\n",
1127
+ " const element = document.querySelector('#df-df099fdf-fd04-4b7a-82e8-3749cda27cda');\n",
1128
+ " const dataTable =\n",
1129
+ " await google.colab.kernel.invokeFunction('convertToInteractive',\n",
1130
+ " [key], {});\n",
1131
+ " if (!dataTable) return;\n",
1132
+ "\n",
1133
+ " const docLinkHtml = 'Like what you see? Visit the ' +\n",
1134
+ " '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
1135
+ " + ' to learn more about interactive tables.';\n",
1136
+ " element.innerHTML = '';\n",
1137
+ " dataTable['output_type'] = 'display_data';\n",
1138
+ " await google.colab.output.renderOutput(dataTable, element);\n",
1139
+ " const docLink = document.createElement('div');\n",
1140
+ " docLink.innerHTML = docLinkHtml;\n",
1141
+ " element.appendChild(docLink);\n",
1142
+ " }\n",
1143
+ " </script>\n",
1144
+ " </div>\n",
1145
+ "\n",
1146
+ "\n",
1147
+ " </div>\n",
1148
+ " </div>\n"
1149
+ ],
1150
+ "application/vnd.google.colaboratory.intrinsic+json": {
1151
+ "type": "dataframe",
1152
+ "variable_name": "df_reviews",
1153
+ "summary": "{\n \"name\": \"df_reviews\",\n \"rows\": 15000,\n \"fields\": [\n {\n \"column\": \"title\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 999,\n \"samples\": [\n \"The Grownup\",\n \"Persepolis: The Story of a Childhood (Persepolis #1-2)\",\n \"Ayumi's Violin\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"sentiment_label\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"neutral\",\n \"negative\",\n \"positive\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"review_text\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 100,\n \"samples\": [\n \"Warm, wise, and wonderfully written. A new favourite.\",\n \"A masterpiece of storytelling. Every page felt purposeful and alive.\",\n \"I can see why others loved it, even if it didn't click for me personally.\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"rating\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 5,\n \"samples\": [\n \"One\",\n \"Two\",\n \"Four\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"popularity_score\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 1,\n \"min\": 1,\n \"max\": 5,\n \"num_unique_values\": 5,\n \"samples\": [\n 2,\n 5,\n 4\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}"
1154
+ }
1155
+ },
1156
+ "metadata": {},
1157
+ "execution_count": 44
1158
+ }
1159
+ ],
1160
+ "source": [
1161
+ "df_reviews.head()"
1162
+ ]
1163
+ }
1164
+ ],
1165
+ "metadata": {
1166
+ "colab": {
1167
+ "collapsed_sections": [
1168
+ "jpASMyIQMaAq",
1169
+ "lquNYCbfL9IM",
1170
+ "0IWuNpxxYDJF",
1171
+ "oCdTsin2Yfp3",
1172
+ "T0TOeRC4Yrnn",
1173
+ "duI5dv3CZYvF",
1174
+ "qMjRKMBQZlJi",
1175
+ "p-1Pr2szaqLk",
1176
+ "SIaJUGIpaH4V",
1177
+ "pY4yCoIuaQqp",
1178
+ "n4-TaNTFgPak",
1179
+ "HnngRNTgacYt",
1180
+ "HF9F9HIzgT7Z",
1181
+ "T8AdKkmASq9a",
1182
+ "OhXbdGD5fH0c",
1183
+ "L2ak1HlcgoTe",
1184
+ "4IXZKcCSgxnq",
1185
+ "EhIjz9WohAmZ",
1186
+ "Gi4y9M9KuDWx",
1187
+ "fQhfVaDmuULT",
1188
+ "bmJMXF-Bukdm",
1189
+ "RYvGyVfXuo54"
1190
+ ],
1191
+ "provenance": []
1192
+ },
1193
+ "kernelspec": {
1194
+ "display_name": "Python 3",
1195
+ "name": "python3"
1196
+ },
1197
+ "language_info": {
1198
+ "name": "python"
1199
+ }
1200
+ },
1201
+ "nbformat": 4,
1202
+ "nbformat_minor": 0
1203
+ }
2a_Python_Analysis (1) (6).ipynb ADDED
The diff for this file is too large to render. See raw diff