Daniel Fried commited on
Commit
bd92b34
1 Parent(s): b43fe4c

update to tokenizers 0.12.1 and use 'debug' file for small model

Browse files
Files changed (3) hide show
  1. .gitignore +1 -1
  2. modules/app.py +7 -7
  3. requirements.txt +1 -1
.gitignore CHANGED
@@ -1,3 +1,3 @@
1
- use_normal_tokenizers
2
  __pycache__
3
  incoder-6B
 
1
+ debug
2
  __pycache__
3
  incoder-6B
modules/app.py CHANGED
@@ -8,19 +8,19 @@ import logging
8
  logging.basicConfig(level=logging.INFO)
9
  import modules.cloud_logging
10
 
 
 
 
 
 
 
11
  # needs to be imported *before* transformers
12
- if os.path.exists('use_normal_tokenizers'):
13
- import tokenizers
14
  BIG_MODEL = False
15
  CUDA = False
16
  else:
17
- import tokenizers_patch
18
  BIG_MODEL = True
19
  CUDA = True
20
- import torch
21
- from transformers import AutoModelForCausalLM, AutoTokenizer
22
- import json
23
- import pprint
24
 
25
  # from flask import Flask, request, render_template
26
  # from flask_cors import CORS
 
8
  logging.basicConfig(level=logging.INFO)
9
  import modules.cloud_logging
10
 
11
+ import tokenizers
12
+ import torch
13
+ from transformers import AutoModelForCausalLM, AutoTokenizer
14
+ import json
15
+ import pprint
16
+
17
  # needs to be imported *before* transformers
18
+ if os.path.exists('debug'):
 
19
  BIG_MODEL = False
20
  CUDA = False
21
  else:
 
22
  BIG_MODEL = True
23
  CUDA = True
 
 
 
 
24
 
25
  # from flask import Flask, request, render_template
26
  # from flask_cors import CORS
requirements.txt CHANGED
@@ -2,6 +2,6 @@ fastapi==0.74.*
2
  requests==2.27.*
3
  torch==1.11.*
4
  uvicorn[standard]==0.17.*
5
- https://dl.fbaipublicfiles.com/codemodels/tokenizers-0.11.6-cp38-cp38-linux_x86_64.whl
6
  git+https://github.com/huggingface/transformers.git@b18dfd95e1f60ae65a959a7b255fc06522170d1b
7
  google-cloud-logging
 
2
  requests==2.27.*
3
  torch==1.11.*
4
  uvicorn[standard]==0.17.*
5
+ tokenizers==0.12.1
6
  git+https://github.com/huggingface/transformers.git@b18dfd95e1f60ae65a959a7b255fc06522170d1b
7
  google-cloud-logging