Commit
•
d288f77
1
Parent(s):
b14d122
Removing cache with logz
Browse files
main.py
CHANGED
|
@@ -54,7 +54,8 @@ def main():
|
|
| 54 |
logger.debug(f"Pushing data for {date} to the Hugging Face hub")
|
| 55 |
dataset.push_to_hub(dataset_name, token=auth_token)
|
| 56 |
logger.info(f"Processed and pushed data for {date} to the Hugging Face Hub")
|
| 57 |
-
dataset.cleanup_cache_files()
|
|
|
|
| 58 |
|
| 59 |
|
| 60 |
def schedule_periodic_task():
|
|
|
|
| 54 |
logger.debug(f"Pushing data for {date} to the Hugging Face hub")
|
| 55 |
dataset.push_to_hub(dataset_name, token=auth_token)
|
| 56 |
logger.info(f"Processed and pushed data for {date} to the Hugging Face Hub")
|
| 57 |
+
files_cleaned = dataset.cleanup_cache_files()
|
| 58 |
+
logger.info(f"Removed {files_cleaned} cache files")
|
| 59 |
|
| 60 |
|
| 61 |
def schedule_periodic_task():
|