Spaces:
Sleeping
Sleeping
update
Browse files- main.py +0 -40
- requirements.txt +0 -2
- toolbox/__init__.py +0 -5
- toolbox/os/__init__.py +0 -6
- toolbox/os/command.py +0 -53
- toolbox/os/environment.py +0 -114
- toolbox/os/other.py +0 -9
main.py
CHANGED
@@ -6,11 +6,7 @@ https://huggingface.co/spaces/sayakpaul/demo-docker-gradio
|
|
6 |
import argparse
|
7 |
import json
|
8 |
import platform
|
9 |
-
from collections import OrderedDict
|
10 |
-
# from typing import OrderedDict
|
11 |
|
12 |
-
from allennlp.models.archival import archive_model, load_archive
|
13 |
-
from allennlp.predictors.text_classifier import TextClassifierPredictor
|
14 |
import fasttext
|
15 |
from fasttext.FastText import load_model, _FastText
|
16 |
import gradio as gr
|
@@ -18,7 +14,6 @@ from gradio import inputs, outputs
|
|
18 |
from langid.langid import LanguageIdentifier, model
|
19 |
|
20 |
from project_settings import project_path, temp_directory
|
21 |
-
from toolbox.os.command import Command
|
22 |
|
23 |
|
24 |
def get_args():
|
@@ -44,34 +39,11 @@ def get_args():
|
|
44 |
|
45 |
lang_id_identifier: LanguageIdentifier = None
|
46 |
fasttext_model: _FastText = None
|
47 |
-
qgyd_lang_id_predictor: TextClassifierPredictor = None
|
48 |
-
|
49 |
-
|
50 |
-
trained_model_dir = project_path / "trained_models/huggingface"
|
51 |
-
trained_model_dir.mkdir(parents=True, exist_ok=True)
|
52 |
-
|
53 |
-
|
54 |
-
def init_qgyd_lang_id_predictor() -> TextClassifierPredictor:
|
55 |
-
model_name = "qgyd2021/language_identification"
|
56 |
-
model_path = trained_model_dir / model_name
|
57 |
-
if not model_path.exists():
|
58 |
-
model_path.parent.mkdir(exist_ok=True)
|
59 |
-
Command.cd(model_path.parent.as_posix())
|
60 |
-
Command.popen("git clone https://huggingface.co/{}".format(model_name))
|
61 |
-
|
62 |
-
archive = load_archive(archive_file=model_path.as_posix())
|
63 |
-
|
64 |
-
predictor = TextClassifierPredictor(
|
65 |
-
model=archive.model,
|
66 |
-
dataset_reader=archive.dataset_reader,
|
67 |
-
)
|
68 |
-
return predictor
|
69 |
|
70 |
|
71 |
def click_lang_id_button(text: str, ground_true: str, model_name: str):
|
72 |
global lang_id_identifier
|
73 |
global fasttext_model
|
74 |
-
global qgyd_lang_id_predictor
|
75 |
|
76 |
text = str(text).strip()
|
77 |
|
@@ -81,16 +53,6 @@ def click_lang_id_button(text: str, ground_true: str, model_name: str):
|
|
81 |
label, prob = fasttext_model.predict(text, k=1)
|
82 |
label = label[0][9:]
|
83 |
prob = prob[0]
|
84 |
-
elif model_name == "qgyd_lang_id_1":
|
85 |
-
json_dict = {
|
86 |
-
"sentence": text
|
87 |
-
}
|
88 |
-
outputs = qgyd_lang_id_predictor.predict_json(
|
89 |
-
json_dict
|
90 |
-
)
|
91 |
-
label = outputs["label"]
|
92 |
-
probs = outputs["probs"]
|
93 |
-
prob = max(probs)
|
94 |
else:
|
95 |
label = "model_name not available."
|
96 |
prob = -1
|
@@ -114,10 +76,8 @@ def main():
|
|
114 |
|
115 |
global lang_id_identifier
|
116 |
global fasttext_model
|
117 |
-
global qgyd_lang_id_predictor
|
118 |
lang_id_identifier = LanguageIdentifier.from_modelstring(model, norm_probs=True)
|
119 |
fasttext_model = fasttext.load_model(args.fasttext_model)
|
120 |
-
qgyd_lang_id_predictor = init_qgyd_lang_id_predictor()
|
121 |
|
122 |
blocks = gr.Interface(
|
123 |
click_lang_id_button,
|
|
|
6 |
import argparse
|
7 |
import json
|
8 |
import platform
|
|
|
|
|
9 |
|
|
|
|
|
10 |
import fasttext
|
11 |
from fasttext.FastText import load_model, _FastText
|
12 |
import gradio as gr
|
|
|
14 |
from langid.langid import LanguageIdentifier, model
|
15 |
|
16 |
from project_settings import project_path, temp_directory
|
|
|
17 |
|
18 |
|
19 |
def get_args():
|
|
|
39 |
|
40 |
lang_id_identifier: LanguageIdentifier = None
|
41 |
fasttext_model: _FastText = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
42 |
|
43 |
|
44 |
def click_lang_id_button(text: str, ground_true: str, model_name: str):
|
45 |
global lang_id_identifier
|
46 |
global fasttext_model
|
|
|
47 |
|
48 |
text = str(text).strip()
|
49 |
|
|
|
53 |
label, prob = fasttext_model.predict(text, k=1)
|
54 |
label = label[0][9:]
|
55 |
prob = prob[0]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
56 |
else:
|
57 |
label = "model_name not available."
|
58 |
prob = -1
|
|
|
76 |
|
77 |
global lang_id_identifier
|
78 |
global fasttext_model
|
|
|
79 |
lang_id_identifier = LanguageIdentifier.from_modelstring(model, norm_probs=True)
|
80 |
fasttext_model = fasttext.load_model(args.fasttext_model)
|
|
|
81 |
|
82 |
blocks = gr.Interface(
|
83 |
click_lang_id_button,
|
requirements.txt
CHANGED
@@ -1,5 +1,3 @@
|
|
1 |
fasttext==0.9.2
|
2 |
langid==1.1.6
|
3 |
gradio==2.3.0
|
4 |
-
allennlp==0.9.0
|
5 |
-
overrides==1.9.0
|
|
|
1 |
fasttext==0.9.2
|
2 |
langid==1.1.6
|
3 |
gradio==2.3.0
|
|
|
|
toolbox/__init__.py
DELETED
@@ -1,5 +0,0 @@
|
|
1 |
-
#!/usr/bin/python3
|
2 |
-
# -*- coding: utf-8 -*-
|
3 |
-
|
4 |
-
if __name__ == "__main__":
|
5 |
-
pass
|
|
|
|
|
|
|
|
|
|
|
|
toolbox/os/__init__.py
DELETED
@@ -1,6 +0,0 @@
|
|
1 |
-
#!/usr/bin/python3
|
2 |
-
# -*- coding: utf-8 -*-
|
3 |
-
|
4 |
-
|
5 |
-
if __name__ == '__main__':
|
6 |
-
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
toolbox/os/command.py
DELETED
@@ -1,53 +0,0 @@
|
|
1 |
-
import os
|
2 |
-
|
3 |
-
|
4 |
-
class Command(object):
|
5 |
-
custom_command = [
|
6 |
-
'cd'
|
7 |
-
]
|
8 |
-
|
9 |
-
@staticmethod
|
10 |
-
def _get_cmd(command):
|
11 |
-
command = str(command).strip()
|
12 |
-
if command == '':
|
13 |
-
return None
|
14 |
-
cmd_and_args = command.split(sep=' ')
|
15 |
-
cmd = cmd_and_args[0]
|
16 |
-
args = ' '.join(cmd_and_args[1:])
|
17 |
-
return cmd, args
|
18 |
-
|
19 |
-
@classmethod
|
20 |
-
def popen(cls, command):
|
21 |
-
cmd, args = cls._get_cmd(command)
|
22 |
-
if cmd in cls.custom_command:
|
23 |
-
method = getattr(cls, cmd)
|
24 |
-
return method(args)
|
25 |
-
else:
|
26 |
-
resp = os.popen(command)
|
27 |
-
result = resp.read()
|
28 |
-
resp.close()
|
29 |
-
return result
|
30 |
-
|
31 |
-
@classmethod
|
32 |
-
def cd(cls, args):
|
33 |
-
if args.startswith('/'):
|
34 |
-
os.chdir(args)
|
35 |
-
else:
|
36 |
-
pwd = os.getcwd()
|
37 |
-
path = os.path.join(pwd, args)
|
38 |
-
os.chdir(path)
|
39 |
-
|
40 |
-
@classmethod
|
41 |
-
def system(cls, command):
|
42 |
-
return os.system(command)
|
43 |
-
|
44 |
-
def __init__(self):
|
45 |
-
pass
|
46 |
-
|
47 |
-
|
48 |
-
def ps_ef_grep(keyword: str):
|
49 |
-
cmd = 'ps -ef | grep {}'.format(keyword)
|
50 |
-
rows = Command.popen(cmd)
|
51 |
-
rows = str(rows).split('\n')
|
52 |
-
rows = [row for row in rows if row.__contains__(keyword) and not row.__contains__('grep')]
|
53 |
-
return rows
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
toolbox/os/environment.py
DELETED
@@ -1,114 +0,0 @@
|
|
1 |
-
#!/usr/bin/python3
|
2 |
-
# -*- coding: utf-8 -*-
|
3 |
-
import json
|
4 |
-
import os
|
5 |
-
|
6 |
-
from dotenv import load_dotenv
|
7 |
-
from dotenv.main import DotEnv
|
8 |
-
|
9 |
-
from smart.json.misc import traverse
|
10 |
-
|
11 |
-
|
12 |
-
class EnvironmentManager(object):
|
13 |
-
def __init__(self, path, env, override=False):
|
14 |
-
filename = os.path.join(path, '{}.env'.format(env))
|
15 |
-
self.filename = filename
|
16 |
-
|
17 |
-
load_dotenv(
|
18 |
-
dotenv_path=filename,
|
19 |
-
override=override
|
20 |
-
)
|
21 |
-
|
22 |
-
self._environ = dict()
|
23 |
-
|
24 |
-
def open_dotenv(self, filename: str = None):
|
25 |
-
filename = filename or self.filename
|
26 |
-
dotenv = DotEnv(
|
27 |
-
dotenv_path=filename,
|
28 |
-
stream=None,
|
29 |
-
verbose=False,
|
30 |
-
interpolate=False,
|
31 |
-
override=False,
|
32 |
-
encoding="utf-8",
|
33 |
-
)
|
34 |
-
result = dotenv.dict()
|
35 |
-
return result
|
36 |
-
|
37 |
-
def get(self, key, default=None, dtype=str):
|
38 |
-
result = os.environ.get(key)
|
39 |
-
if result is None:
|
40 |
-
if default is None:
|
41 |
-
result = None
|
42 |
-
else:
|
43 |
-
result = default
|
44 |
-
else:
|
45 |
-
result = dtype(result)
|
46 |
-
self._environ[key] = result
|
47 |
-
return result
|
48 |
-
|
49 |
-
|
50 |
-
_DEFAULT_DTYPE_MAP = {
|
51 |
-
'int': int,
|
52 |
-
'float': float,
|
53 |
-
'str': str,
|
54 |
-
'json.loads': json.loads
|
55 |
-
}
|
56 |
-
|
57 |
-
|
58 |
-
class JsonConfig(object):
|
59 |
-
"""
|
60 |
-
将 json 中, 形如 `$float:threshold` 的值, 处理为:
|
61 |
-
从环境变量中查到 threshold, 再将其转换为 float 类型.
|
62 |
-
"""
|
63 |
-
def __init__(self, dtype_map: dict = None, environment: EnvironmentManager = None):
|
64 |
-
self.dtype_map = dtype_map or _DEFAULT_DTYPE_MAP
|
65 |
-
self.environment = environment or os.environ
|
66 |
-
|
67 |
-
def sanitize_by_filename(self, filename: str):
|
68 |
-
with open(filename, 'r', encoding='utf-8') as f:
|
69 |
-
js = json.load(f)
|
70 |
-
|
71 |
-
return self.sanitize_by_json(js)
|
72 |
-
|
73 |
-
def sanitize_by_json(self, js):
|
74 |
-
js = traverse(
|
75 |
-
js,
|
76 |
-
callback=self.sanitize,
|
77 |
-
environment=self.environment
|
78 |
-
)
|
79 |
-
return js
|
80 |
-
|
81 |
-
def sanitize(self, string, environment):
|
82 |
-
"""支持 $ 符开始的, 环境变量配置"""
|
83 |
-
if isinstance(string, str) and string.startswith('$'):
|
84 |
-
dtype, key = string[1:].split(':')
|
85 |
-
dtype = self.dtype_map[dtype]
|
86 |
-
|
87 |
-
value = environment.get(key)
|
88 |
-
if value is None:
|
89 |
-
raise AssertionError('environment not exist. key: {}'.format(key))
|
90 |
-
|
91 |
-
value = dtype(value)
|
92 |
-
result = value
|
93 |
-
else:
|
94 |
-
result = string
|
95 |
-
return result
|
96 |
-
|
97 |
-
|
98 |
-
def demo1():
|
99 |
-
import json
|
100 |
-
|
101 |
-
from project_settings import project_path
|
102 |
-
|
103 |
-
environment = EnvironmentManager(
|
104 |
-
path=os.path.join(project_path, 'server/callbot_server/dotenv'),
|
105 |
-
env='dev',
|
106 |
-
)
|
107 |
-
init_scenes = environment.get(key='init_scenes', dtype=json.loads)
|
108 |
-
print(init_scenes)
|
109 |
-
print(environment._environ)
|
110 |
-
return
|
111 |
-
|
112 |
-
|
113 |
-
if __name__ == '__main__':
|
114 |
-
demo1()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
toolbox/os/other.py
DELETED
@@ -1,9 +0,0 @@
|
|
1 |
-
import os
|
2 |
-
import inspect
|
3 |
-
|
4 |
-
|
5 |
-
def pwd():
|
6 |
-
"""你在哪个文件调用此函数, 它就会返回那个文件所在的 dir 目标"""
|
7 |
-
frame = inspect.stack()[1]
|
8 |
-
module = inspect.getmodule(frame[0])
|
9 |
-
return os.path.dirname(os.path.abspath(module.__file__))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|