Clémentine commited on
Commit
bde3a6f
1 Parent(s): d084b26

fix tokenizer detection

Browse files
Files changed (1) hide show
  1. src/submission/check_validity.py +1 -1
src/submission/check_validity.py CHANGED
@@ -42,7 +42,7 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_rem
42
  config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
43
  if test_tokenizer:
44
  tokenizer_config = get_tokenizer_config(model_name)
45
- if tokenizer_config is not None:
46
  tokenizer_class_candidate = tokenizer_config.get("tokenizer_class", None)
47
  else:
48
  tokenizer_class_candidate = config.tokenizer_class
 
42
  config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
43
  if test_tokenizer:
44
  tokenizer_config = get_tokenizer_config(model_name)
45
+ if tokenizer_config not in [None, {}]:
46
  tokenizer_class_candidate = tokenizer_config.get("tokenizer_class", None)
47
  else:
48
  tokenizer_class_candidate = config.tokenizer_class