Clémentine commited on
Commit
8137036
1 Parent(s): a99162a

Fixed tokenizer message

Browse files
Files changed (1) hide show
  1. src/submission/check_validity.py +3 -1
src/submission/check_validity.py CHANGED
@@ -42,13 +42,15 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_rem
42
  config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
43
  if test_tokenizer:
44
  try:
45
- AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
46
  except ValueError as e:
47
  return (
48
  False,
49
  f"uses a tokenizer which is not in a transformers release: {e}",
50
  None
51
  )
 
 
52
  return True, None, config
53
 
54
  except ValueError:
 
42
  config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
43
  if test_tokenizer:
44
  try:
45
+ tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
46
  except ValueError as e:
47
  return (
48
  False,
49
  f"uses a tokenizer which is not in a transformers release: {e}",
50
  None
51
  )
52
+ except Exception as e:
53
+ return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
54
  return True, None, config
55
 
56
  except ValueError: