Spaces:
Sleeping
Sleeping
Update app.py
Browse fileszip folder download
app.py
CHANGED
@@ -13,7 +13,7 @@ NTU_BLUE = "#003D7C"
|
|
13 |
NTU_RED = "#C11E38"
|
14 |
NTU_GOLD = "#E7B820"
|
15 |
|
16 |
-
def process_data(file: gr.File, progress=gr.Progress()) -> Tuple[str, str, pd.DataFrame, Union[
|
17 |
try:
|
18 |
# Check if file is uploaded
|
19 |
if file is None:
|
@@ -29,14 +29,8 @@ def process_data(file: gr.File, progress=gr.Progress()) -> Tuple[str, str, pd.Da
|
|
29 |
except Exception as e:
|
30 |
raise ValueError(f"Error reading Excel file: {str(e)}")
|
31 |
|
32 |
-
|
33 |
-
|
34 |
-
missing_columns = [col for col in required_columns if col not in raw_data.columns]
|
35 |
-
if missing_columns:
|
36 |
-
raise ValueError(f"Missing required columns: {', '.join(missing_columns)}")
|
37 |
-
|
38 |
-
# Extract filename without extension
|
39 |
-
base_filename = os.path.splitext(os.path.basename(file.name))[0]
|
40 |
|
41 |
# Step 1: Extract User Information
|
42 |
user_info = raw_data[['user_id', 'lastname']].drop_duplicates().copy()
|
@@ -60,9 +54,9 @@ def process_data(file: gr.File, progress=gr.Progress()) -> Tuple[str, str, pd.Da
|
|
60 |
|
61 |
progress(0.6, desc="Calculating grand totals")
|
62 |
|
63 |
-
# Step 4: Generate Filenames and Paths
|
64 |
user_info['File'] = 'User_' + user_info['Username'] + '_data.csv'
|
65 |
-
user_info['Path'] = '
|
66 |
|
67 |
# Remove extra columns and summary rows
|
68 |
user_info = user_info[['Username', 'Name', 'Courses', 'Grand Total', 'Email', 'File', 'Path']]
|
@@ -72,33 +66,53 @@ def process_data(file: gr.File, progress=gr.Progress()) -> Tuple[str, str, pd.Da
|
|
72 |
|
73 |
progress(0.8, desc="Generating output files")
|
74 |
|
75 |
-
#
|
76 |
-
|
|
|
77 |
|
78 |
-
|
79 |
-
|
80 |
-
|
81 |
-
|
82 |
-
|
83 |
-
|
84 |
-
|
85 |
-
|
86 |
-
|
87 |
-
|
88 |
-
|
89 |
-
|
90 |
-
|
91 |
-
|
92 |
-
|
93 |
-
|
94 |
-
worksheet.write(f'C{last_row + 1}', user_info['Courses'].sum())
|
95 |
-
worksheet.write(f'D{last_row + 1}', user_info['Grand Total'].sum())
|
96 |
|
97 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
98 |
|
99 |
zip_buffer.seek(0)
|
|
|
|
|
|
|
|
|
|
|
100 |
progress(1.0, desc="Processing complete")
|
101 |
-
return "Processing complete. You can now download the results.", "Results are packaged in the zip file below.", user_info,
|
102 |
except Exception as e:
|
103 |
error_msg = f"Error: {str(e)}\n\nTraceback:\n{traceback.format_exc()}"
|
104 |
return error_msg, "Processing failed", pd.DataFrame(), None
|
@@ -148,7 +162,7 @@ def create_scatter_plot(df: pd.DataFrame) -> Union[px.scatter, None]:
|
|
148 |
print(f"Error creating scatter plot: {str(e)}")
|
149 |
return None
|
150 |
|
151 |
-
def update_insights(df: pd.DataFrame,
|
152 |
try:
|
153 |
if df.empty:
|
154 |
return [gr.Markdown("No data available. Please upload and process a file first.")] + [None] * 6
|
@@ -162,9 +176,8 @@ def update_insights(df: pd.DataFrame, zip_data: Union[Tuple[io.BytesIO, str], No
|
|
162 |
|
163 |
user_table = gr.DataFrame(value=df)
|
164 |
|
165 |
-
if
|
166 |
-
|
167 |
-
download_button = gr.File(value=zip_file, filename=zip_name, visible=True, label="Download Results")
|
168 |
download_text = gr.Markdown("Click the 'Download Results' button above to download the ZIP file containing all processed data.")
|
169 |
else:
|
170 |
download_button = gr.File(visible=False, label="Download Results")
|
@@ -177,12 +190,12 @@ def update_insights(df: pd.DataFrame, zip_data: Union[Tuple[io.BytesIO, str], No
|
|
177 |
|
178 |
def process_and_update(file):
|
179 |
try:
|
180 |
-
result_msg, csv_loc, df,
|
181 |
-
insights = update_insights(df,
|
182 |
-
return [result_msg, csv_loc] + insights
|
183 |
except Exception as e:
|
184 |
error_msg = f"Error in process_and_update: {str(e)}\n\nTraceback:\n{traceback.format_exc()}"
|
185 |
-
return [error_msg, "Processing failed"] + [gr.Markdown(error_msg)] + [None] * 6
|
186 |
|
187 |
def clear_outputs():
|
188 |
return [""] * 2 + [None] * 6 + [""] # 2 text outputs, 6 graph/table/file outputs, and 1 download text
|
@@ -201,40 +214,7 @@ custom_theme = gr.themes.Base().set(
|
|
201 |
input_border_color_focus="#C11E38",
|
202 |
)
|
203 |
|
204 |
-
|
205 |
-
custom_css = """
|
206 |
-
.gr-button-secondary {
|
207 |
-
background-color: #F0F0F0;
|
208 |
-
color: #003D7C;
|
209 |
-
border: 1px solid #003D7C;
|
210 |
-
border-radius: 12px;
|
211 |
-
padding: 8px 16px;
|
212 |
-
font-size: 16px;
|
213 |
-
font-weight: bold;
|
214 |
-
cursor: pointer;
|
215 |
-
transition: background-color 0.3s, color 0.3s, border-color 0.3s;
|
216 |
-
}
|
217 |
-
|
218 |
-
.gr-button-secondary:hover {
|
219 |
-
background-color: #003D7C;
|
220 |
-
color: white;
|
221 |
-
border-color: #003D7C;
|
222 |
-
}
|
223 |
-
|
224 |
-
.gr-button-secondary:active {
|
225 |
-
transform: translateY(1px);
|
226 |
-
}
|
227 |
-
|
228 |
-
.app-title {
|
229 |
-
color: #003D7C;
|
230 |
-
font-size: 24px;
|
231 |
-
font-weight: bold;
|
232 |
-
text-align: center;
|
233 |
-
margin-bottom: 20px;
|
234 |
-
}
|
235 |
-
"""
|
236 |
-
|
237 |
-
with gr.Blocks(theme=custom_theme, css=custom_css) as iface:
|
238 |
gr.Markdown("# Gradebook Data Processor", elem_classes=["app-title"])
|
239 |
|
240 |
with gr.Tabs():
|
@@ -244,6 +224,7 @@ with gr.Blocks(theme=custom_theme, css=custom_css) as iface:
|
|
244 |
process_btn = gr.Button("Process Data", variant="primary")
|
245 |
output_msg = gr.Textbox(label="Processing Result")
|
246 |
csv_location = gr.Textbox(label="Output Information")
|
|
|
247 |
gr.Markdown("After processing, switch to the 'Data Insights' tab to view results and download files.")
|
248 |
|
249 |
with gr.TabItem("2. Data Insights Dashboard"):
|
@@ -256,10 +237,6 @@ with gr.Blocks(theme=custom_theme, css=custom_css) as iface:
|
|
256 |
|
257 |
scatter_plot = gr.Plot()
|
258 |
user_table = gr.DataFrame()
|
259 |
-
|
260 |
-
gr.Markdown("## Download Processed Data")
|
261 |
-
download_button = gr.File(visible=False, label="Download Results")
|
262 |
-
download_text = gr.Markdown("")
|
263 |
|
264 |
clear_btn = gr.Button("Clear All Data", variant="secondary")
|
265 |
gr.Markdown("Click 'Clear All Data' to reset the application and start over.")
|
@@ -267,14 +244,14 @@ with gr.Blocks(theme=custom_theme, css=custom_css) as iface:
|
|
267 |
process_btn.click(
|
268 |
process_and_update,
|
269 |
inputs=[file_input],
|
270 |
-
outputs=[output_msg, csv_location, summary_stats, users_activity_chart, users_courses_chart, scatter_plot, user_table, download_button
|
271 |
)
|
272 |
|
273 |
clear_btn.click(
|
274 |
clear_outputs,
|
275 |
inputs=[],
|
276 |
-
outputs=[output_msg, csv_location, summary_stats, users_activity_chart, users_courses_chart, scatter_plot, user_table, download_button
|
277 |
)
|
278 |
|
279 |
if __name__ == "__main__":
|
280 |
-
iface.launch()
|
|
|
13 |
NTU_RED = "#C11E38"
|
14 |
NTU_GOLD = "#E7B820"
|
15 |
|
16 |
+
def process_data(file: gr.File, progress=gr.Progress()) -> Tuple[str, str, pd.DataFrame, Union[str, None]]:
|
17 |
try:
|
18 |
# Check if file is uploaded
|
19 |
if file is None:
|
|
|
29 |
except Exception as e:
|
30 |
raise ValueError(f"Error reading Excel file: {str(e)}")
|
31 |
|
32 |
+
base_path = tempfile.mkdtemp()
|
33 |
+
final_file_path = os.path.join(base_path, 'final_output.xlsx')
|
|
|
|
|
|
|
|
|
|
|
|
|
34 |
|
35 |
# Step 1: Extract User Information
|
36 |
user_info = raw_data[['user_id', 'lastname']].drop_duplicates().copy()
|
|
|
54 |
|
55 |
progress(0.6, desc="Calculating grand totals")
|
56 |
|
57 |
+
# Step 4: Generate Filenames and Paths
|
58 |
user_info['File'] = 'User_' + user_info['Username'] + '_data.csv'
|
59 |
+
user_info['Path'] = user_info['File'].apply(lambda x: os.path.join(base_path, 'mailmerge', x))
|
60 |
|
61 |
# Remove extra columns and summary rows
|
62 |
user_info = user_info[['Username', 'Name', 'Courses', 'Grand Total', 'Email', 'File', 'Path']]
|
|
|
66 |
|
67 |
progress(0.8, desc="Generating output files")
|
68 |
|
69 |
+
# Calculate totals for Courses and Grand Total
|
70 |
+
total_courses = user_info['Courses'].sum()
|
71 |
+
total_grand_total = user_info['Grand Total'].sum()
|
72 |
|
73 |
+
# Generate individual CSV files for each user
|
74 |
+
required_columns = ['course_id', 'course_pk1', 'data', 'event_type', 'internal_handle', 'lastname', 'session_id', 'timestamp', 'user_id', 'system_role']
|
75 |
+
mailmerge_path = os.path.join(base_path, 'mailmerge')
|
76 |
+
if not os.path.exists(mailmerge_path):
|
77 |
+
os.makedirs(mailmerge_path)
|
78 |
+
|
79 |
+
for user_id in user_info['Username'].unique():
|
80 |
+
user_data = raw_data[raw_data['user_id'] == user_id][required_columns]
|
81 |
+
user_file_path = os.path.join(mailmerge_path, f'User_{user_id}_data.csv')
|
82 |
+
user_data.to_csv(user_file_path, index=False)
|
83 |
+
|
84 |
+
# Save the final dataframe to the output Excel file
|
85 |
+
with pd.ExcelWriter(final_file_path, engine='xlsxwriter') as writer:
|
86 |
+
user_info.to_excel(writer, index=False, sheet_name='Sheet1')
|
87 |
+
workbook = writer.book
|
88 |
+
worksheet = writer.sheets['Sheet1']
|
|
|
|
|
89 |
|
90 |
+
# Find the last row number dynamically
|
91 |
+
last_row = len(user_info) + 1 # Account for header row in Excel
|
92 |
+
|
93 |
+
# Write the total values in columns B, C, and D of the first empty row after the user data
|
94 |
+
worksheet.write(f'B{last_row + 1}', 'Total')
|
95 |
+
worksheet.write(f'C{last_row + 1}', total_courses)
|
96 |
+
worksheet.write(f'D{last_row + 1}', total_grand_total)
|
97 |
+
|
98 |
+
# Create a zip file containing all user CSV files and the final Excel file
|
99 |
+
zip_buffer = io.BytesIO()
|
100 |
+
with zipfile.ZipFile(zip_buffer, 'w', zipfile.ZIP_DEFLATED) as zip_file:
|
101 |
+
for root, _, files in os.walk(mailmerge_path):
|
102 |
+
for file in files:
|
103 |
+
file_path = os.path.join(root, file)
|
104 |
+
zip_file.write(file_path, os.path.relpath(file_path, base_path))
|
105 |
+
|
106 |
+
zip_file.write(final_file_path, os.path.basename(final_file_path))
|
107 |
|
108 |
zip_buffer.seek(0)
|
109 |
+
|
110 |
+
temp_zip_file = tempfile.NamedTemporaryFile(delete=False, suffix=".zip")
|
111 |
+
with open(temp_zip_file.name, 'wb') as f:
|
112 |
+
f.write(zip_buffer.getvalue())
|
113 |
+
|
114 |
progress(1.0, desc="Processing complete")
|
115 |
+
return "Processing complete. You can now download the results.", "Results are packaged in the zip file below.", user_info, temp_zip_file.name
|
116 |
except Exception as e:
|
117 |
error_msg = f"Error: {str(e)}\n\nTraceback:\n{traceback.format_exc()}"
|
118 |
return error_msg, "Processing failed", pd.DataFrame(), None
|
|
|
162 |
print(f"Error creating scatter plot: {str(e)}")
|
163 |
return None
|
164 |
|
165 |
+
def update_insights(df: pd.DataFrame, zip_path: Union[str, None]) -> List[Union[gr.components.Component, None]]:
|
166 |
try:
|
167 |
if df.empty:
|
168 |
return [gr.Markdown("No data available. Please upload and process a file first.")] + [None] * 6
|
|
|
176 |
|
177 |
user_table = gr.DataFrame(value=df)
|
178 |
|
179 |
+
if zip_path:
|
180 |
+
download_button = gr.File(value=zip_path, visible=True, label="Download Results")
|
|
|
181 |
download_text = gr.Markdown("Click the 'Download Results' button above to download the ZIP file containing all processed data.")
|
182 |
else:
|
183 |
download_button = gr.File(visible=False, label="Download Results")
|
|
|
190 |
|
191 |
def process_and_update(file):
|
192 |
try:
|
193 |
+
result_msg, csv_loc, df, zip_path = process_data(file)
|
194 |
+
insights = update_insights(df, zip_path)
|
195 |
+
return [result_msg, csv_loc, zip_path] + insights
|
196 |
except Exception as e:
|
197 |
error_msg = f"Error in process_and_update: {str(e)}\n\nTraceback:\n{traceback.format_exc()}"
|
198 |
+
return [error_msg, "Processing failed", None] + [gr.Markdown(error_msg)] + [None] * 6
|
199 |
|
200 |
def clear_outputs():
|
201 |
return [""] * 2 + [None] * 6 + [""] # 2 text outputs, 6 graph/table/file outputs, and 1 download text
|
|
|
214 |
input_border_color_focus="#C11E38",
|
215 |
)
|
216 |
|
217 |
+
with gr.Blocks(theme=custom_theme, css="custom.css") as iface:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
218 |
gr.Markdown("# Gradebook Data Processor", elem_classes=["app-title"])
|
219 |
|
220 |
with gr.Tabs():
|
|
|
224 |
process_btn = gr.Button("Process Data", variant="primary")
|
225 |
output_msg = gr.Textbox(label="Processing Result")
|
226 |
csv_location = gr.Textbox(label="Output Information")
|
227 |
+
download_button = gr.File(visible=False, label="Download Results")
|
228 |
gr.Markdown("After processing, switch to the 'Data Insights' tab to view results and download files.")
|
229 |
|
230 |
with gr.TabItem("2. Data Insights Dashboard"):
|
|
|
237 |
|
238 |
scatter_plot = gr.Plot()
|
239 |
user_table = gr.DataFrame()
|
|
|
|
|
|
|
|
|
240 |
|
241 |
clear_btn = gr.Button("Clear All Data", variant="secondary")
|
242 |
gr.Markdown("Click 'Clear All Data' to reset the application and start over.")
|
|
|
244 |
process_btn.click(
|
245 |
process_and_update,
|
246 |
inputs=[file_input],
|
247 |
+
outputs=[output_msg, csv_location, download_button, summary_stats, users_activity_chart, users_courses_chart, scatter_plot, user_table, download_button]
|
248 |
)
|
249 |
|
250 |
clear_btn.click(
|
251 |
clear_outputs,
|
252 |
inputs=[],
|
253 |
+
outputs=[output_msg, csv_location, summary_stats, users_activity_chart, users_courses_chart, scatter_plot, user_table, download_button]
|
254 |
)
|
255 |
|
256 |
if __name__ == "__main__":
|
257 |
+
iface.launch()
|