ClΓ©mentine commited on
Commit
5933808
β€’
1 Parent(s): e616538

updated caching to include size of models

Browse files
model_info_cache.pkl CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ffc617fbd884d6c7e5c8cef9fc815c2b7e88cf578ec53954bd5b81fdfdad21fc
3
- size 2983762
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c80b745050df96eb1bc908e15b2406533b076c9160486a48b88c8a29f1ed312
3
+ size 2985167
model_size_cache.pkl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5b09d9f81d22f7849f92081950b675c2d68e3bfd320e5dfd1892d14602a29a2
3
+ size 58166
src/display_models/get_model_metadata.py CHANGED
@@ -23,8 +23,13 @@ def get_model_infos_from_hub(leaderboard_data: List[dict]):
23
  try:
24
  with open("model_info_cache.pkl", "rb") as f:
25
  model_info_cache = pickle.load(f)
26
- except EOFError:
27
  model_info_cache = {}
 
 
 
 
 
28
 
29
  for model_data in tqdm(leaderboard_data):
30
  model_name = model_data["model_name_for_query"]
@@ -39,16 +44,21 @@ def get_model_infos_from_hub(leaderboard_data: List[dict]):
39
  print("Repo not found!", model_name)
40
  model_data[AutoEvalColumn.license.name] = None
41
  model_data[AutoEvalColumn.likes.name] = None
42
- model_data[AutoEvalColumn.params.name] = get_model_size(model_name, None)
43
- continue
 
44
 
45
  model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
46
  model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
47
- model_data[AutoEvalColumn.params.name] = get_model_size(model_name, model_info)
 
 
48
 
49
  # save cache to disk in pickle format
50
  with open("model_info_cache.pkl", "wb") as f:
51
  pickle.dump(model_info_cache, f)
 
 
52
 
53
 
54
  def get_model_license(model_info):
 
23
  try:
24
  with open("model_info_cache.pkl", "rb") as f:
25
  model_info_cache = pickle.load(f)
26
+ except (EOFError, FileNotFoundError):
27
  model_info_cache = {}
28
+ try:
29
+ with open("model_size_cache.pkl", "rb") as f:
30
+ model_size_cache = pickle.load(f)
31
+ except (EOFError, FileNotFoundError):
32
+ model_size_cache = {}
33
 
34
  for model_data in tqdm(leaderboard_data):
35
  model_name = model_data["model_name_for_query"]
 
44
  print("Repo not found!", model_name)
45
  model_data[AutoEvalColumn.license.name] = None
46
  model_data[AutoEvalColumn.likes.name] = None
47
+ if model_name not in model_size_cache:
48
+ model_size_cache[model_name] = get_model_size(model_name, None)
49
+ model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
50
 
51
  model_data[AutoEvalColumn.license.name] = get_model_license(model_info)
52
  model_data[AutoEvalColumn.likes.name] = get_model_likes(model_info)
53
+ if model_name not in model_size_cache:
54
+ model_size_cache[model_name] = get_model_size(model_name, model_info)
55
+ model_data[AutoEvalColumn.params.name] = model_size_cache[model_name]
56
 
57
  # save cache to disk in pickle format
58
  with open("model_info_cache.pkl", "wb") as f:
59
  pickle.dump(model_info_cache, f)
60
+ with open("model_size_cache.pkl", "wb") as f:
61
+ pickle.dump(model_size_cache, f)
62
 
63
 
64
  def get_model_license(model_info):