update v1.0.1

.gitignore

@@ -1 +1,2 @@
-tmp
+tmp
+__pycache__

google_sheet.py

@@ -51,10 +51,11 @@ def create_repositories():
 
     config_repository = SheetCRUDRepository(sheet.get_worksheet(0))
     log_repository = SheetCRUDRepository(sheet.get_worksheet(1))
-    return config_repository, log_repository
+    secret_repository = SheetCRUDRepository(sheet.get_worksheet(2))
+    return config_repository, log_repository, secret_repository
 
 
-conf_repo, log_repo = create_repositories()
+conf_repo, log_repo, secret_repo = create_repositories()
 
 
 if __name__ == "__main__":

kaggle_service.py

@@ -44,6 +44,83 @@ class KaggleApiWrapper(KaggleApi):
         # todo: fix bug when delete api
         pass
 
+#     def get_accelerator_quota_with_http_info(self):  # noqa: E501
+#         """
+#
+#         This method makes a synchronous HTTP request by default. To make an
+#         asynchronous HTTP request, please pass async_req=True
+#         >>> thread = api.competitions_list_with_http_info(async_req=True)
+#         >>> result = thread.get()
+#
+#         :param async_req bool
+#         :param str group: Filter competitions by a particular group
+#         :param str category: Filter competitions by a particular category
+#         :param str sort_by: Sort the results
+#         :param int page: Page number
+#         :param str search: Search terms
+#         :return: Result
+#                  If the method is called asynchronously,
+#                  returns the request thread.
+#         """
+#
+#         all_params = []  # noqa: E501
+#         all_params.append('async_req')
+#         all_params.append('_return_http_data_only')
+#         all_params.append('_preload_content')
+#         all_params.append('_request_timeout')
+#
+#         params = locals()
+#
+#         collection_formats = {}
+#
+#         path_params = {}
+#
+#         query_params = []
+#         # if 'group' in params:
+#         #     query_params.append(('group', params['group']))  # noqa: E501
+#         # if 'category' in params:
+#         #     query_params.append(('category', params['category']))  # noqa: E501
+#         # if 'sort_by' in params:
+#         #     query_params.append(('sortBy', params['sort_by']))  # noqa: E501
+#         # if 'page' in params:
+#         #     query_params.append(('page', params['page']))  # noqa: E501
+#         # if 'search' in params:
+#         #     query_params.append(('search', params['search']))  # noqa: E501
+#
+#         header_params = {}
+#
+#         form_params = []
+#         local_var_files = {}
+#
+#         body_params = None
+#         # HTTP header `Accept`
+#         header_params['Accept'] = self.api_client.select_header_accept(
+#             ['application/json'])  # noqa: E501
+#
+#         # Authentication setting
+#         auth_settings = ['basicAuth']  # noqa: E501
+#
+#         return self.api_client.call_api(
+#             'i/kernels.KernelsService/GetAcceleratorQuotaStatistics', 'GET',
+#             # '/competitions/list', 'GET',
+#             path_params,
+#             query_params,
+#             header_params,
+#             body=body_params,
+#             post_params=form_params,
+#             files=local_var_files,
+#             response_type='Result',  # noqa: E501
+#             auth_settings=auth_settings,
+#             async_req=params.get('async_req'),
+#             _return_http_data_only=params.get('_return_http_data_only'),
+#             _preload_content=params.get('_preload_content', True),
+#             _request_timeout=params.get('_request_timeout'),
+#             collection_formats=collection_formats)
+#
+# if __name__ == "__main__":
+#     api = KaggleApiWrapper('ha.vt194547@sis.hust.edu.vn', "c54e96568075fcc277bd10ba0e0a52b9")
+#     api.authenticate()
+#     print(api.get_accelerator_quota_with_http_info())
 
 class ValidateException(Exception):
     def __init__(self, message: str):
@@ -309,14 +386,16 @@ class NbJob:
                 sheet_logger.log(username=api.username, nb=self.nb_slug, log="Try but no effect. Seem account to be out of quota")
                 return False
             if status3 in self.not_rerun_stt:
-                sheet_logger.log(username=api.username, nb=self.nb_slug, log=f"Notebook status is {status3} is in ignore status list {self.not_rerun_stt}, do nothing!")
+                # sheet_logger.log(username=api.username, nb=self.nb_slug, log=f"Notebook status is {status3} is in ignore status list {self.not_rerun_stt}, do nothing!")
+                sheet_logger.log(username=api.username, nb=self.nb_slug,
+                                 log=f"Schedule notebook successfully. Current status is '{status3}'")
                 return True
             if status3 not in ["queued", "running"]:
                 # return False # todo: check when user is out of quota
                 print(f"Error: status is {status3}")
+
                 raise Exception("Setup exception")
-            sheet_logger.log(username=api.username, nb=self.nb_slug,
-                             log=f"Schedule notebook successfully. Current status is '{status3}'")
+
             return True
 
         sheet_logger.log(username=api.username, nb=self.nb_slug, log=f"Notebook status is '{status1}' is not in {self.rerun_stt}, do nothing!")
@@ -428,16 +507,16 @@ class KernelRerunService:
             print(f"Job: {job.nb_slug} {success}")
 
 
-if __name__ == "__main__":
-    service = KernelRerunService()
-    files = os.listdir("./config")
-    for file in files:
-        if '.example' not in file:
-            with open(os.path.join("./config", file), "r") as f:
-                obj = json.loads(f.read())
-                print(obj)
-                service.add_job(NbJob.from_dict(obj))
-    service.run_all()
+# if __name__ == "__main__":
+    # service = KernelRerunService()
+    # files = os.listdir("./config")
+    # for file in files:
+    #     if '.example' not in file:
+    #         with open(os.path.join("./config", file), "r") as f:
+    #             obj = json.loads(f.read())
+    #             print(obj)
+    #             service.add_job(NbJob.from_dict(obj))
+    # service.run_all()
 
     # try:
     #     acc_secret_dict = {

logger.py

@@ -1,8 +1,7 @@
 import platform,socket,re,uuid,json,psutil,logging
 from datetime import datetime as dt
 from google_sheet import log_repo, conf_repo, SheetCRUDRepository
-
-version="v1.0.0"
+from version import VERSION
 
 
 def get_sys_info():
@@ -34,7 +33,7 @@ class SheetLogger:
             "kaggle_username": username,
             "log": log,
             "device": str(get_sys_info()),
-            "version": version
+            "version": VERSION
         })
 
     def update_job_status(self, row, validate_status: str = None, notebook_status: str = None):

main.py

@@ -1,4 +1,4 @@
-from google_sheet import conf_repo
+from google_sheet import conf_repo, secret_repo
 import json
 
 from kaggle_service import KernelRerunService, NbJob
@@ -6,23 +6,81 @@ from logger import sheet_logger
 
 
 if __name__ == "__main__":
+    sheet_logger.log("========= start ==========")
+
+    # load all account and secret into a dict
+    secret_dict = {}
+    try:
+        for i in range(2, 50):  # note: read row 2 - 50
+            rs = secret_repo.read(i)
+            if not rs:
+                break
+            assert 'username' in rs
+            assert 'secret' in rs
+
+            username = rs['username'].rstrip()
+            secret = rs['secret'].rstrip()
+
+            secret_dict[username] = secret
+    except Exception as e:
+        sheet_logger.log(log="Get secret table failed!!" + str(e))
+
+    # load config from gg sheet file
     configs = []
     ids = []
     try:
-        for i in range(2, 256):
+        for i in range(2, 50):     # note: read row 2 - 50
             rs = conf_repo.read(i)
             if not rs:
                 break
+            if not rs['config']:
+                reset_keys = []
+                for key in rs:
+                    if rs[key]:
+                        reset_keys.append(key)
+                if len(reset_keys) > 0:
+                    conf_repo.update(row_index=i, data={k: '' for k in reset_keys})
+                break
+            print(i, rs)
             try:
+                # validation
+                # assert 'config' in rs, 'require config column'
+                assert 'usernames' in rs, 'require usernames column'
+                assert 'enable' in rs, 'require enable column'
+
+                assert rs['enable'].rstrip() in ['enable', 'disable'], rs['enable']
+                assert 'slug' in rs['config'], 'require config.slug'
+
                 cfg = json.loads(rs['config'])
+                usernames = set(rs['usernames']
+                             .rstrip().replace('\n', ' ').replace(',', ' ').replace(';', ' ').replace('|', ' ')
+                             .split(' '))
+                usernames = [u for u in usernames if u]
+                is_enabled = rs['enable'].rstrip() == 'enable'
+                if not is_enabled:
+                    sheet_logger.log(log="Disabled, skip!", nb=cfg['slug'])
+                    print("skip ", cfg['slug'])
+                    continue
+
+                assert len(usernames) > 0, 'len usernames == 0'
+
+                # process config
+                accounts = {u: secret_dict[u] for u in secret_dict if u in usernames}
+                assert not set(usernames).difference(set(accounts.keys())), set(usernames).difference(set(accounts.keys()))
+                cfg = {**cfg, "accounts": accounts}
+
+                # save
                 configs.append(cfg)
                 ids.append(i)
                 print(cfg)
-            except Exception as e:
-                sheet_logger.update_job_status(i, validate_status=str(e))
-    except Exception as e:
-        sheet_logger.log(log="Get config failed!!" + str(e))
+            except AssertionError:
+                import traceback
+                sheet_logger.update_job_status(i, validate_status=str(traceback.format_exc()))
+    except Exception:
+        import traceback
+        sheet_logger.log(log="Get config failed!!" + str(traceback.format_exc()))
 
+    # add config to service
     service = KernelRerunService()
     for config, idx in zip(configs, ids):
         try:
@@ -30,6 +88,7 @@ if __name__ == "__main__":
         except Exception as e:
             sheet_logger.update_job_status(idx, validate_status=str(e))
 
+    # run service
     try:
         service.validate_all()
         # service.status_all()
@@ -37,3 +96,4 @@ if __name__ == "__main__":
     except Exception as e:
         sheet_logger.log(log=str(e))
 
+    sheet_logger.log("========= end ==========")

version.py

@@ -0,0 +1 @@
+VERSION="v1.0.1"