raynardj commited on
Commit
9834964
1 Parent(s): 5e68d54

🪕 baseline

Browse files
Files changed (3) hide show
  1. .gitignore +1 -0
  2. app.py +172 -0
  3. meta.csv +0 -0
.gitignore ADDED
@@ -0,0 +1 @@
 
1
+ .streamlit/*
app.py ADDED
@@ -0,0 +1,172 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import streamlit as st
2
+ import pandas as pd
3
+ from pathlib import Path
4
+ import requests
5
+ import base64
6
+ from requests.auth import HTTPBasicAuth
7
+ import torch
8
+
9
+ st.set_page_config(layout="wide")
10
+ st.title("【随无涯】")
11
+
12
+ @st.cache(allow_output_mutation=True)
13
+ def load_model():
14
+ from transformers import (
15
+ EncoderDecoderModel,
16
+ AutoTokenizer
17
+ )
18
+ PRETRAINED = "raynardj/wenyanwen-ancient-translate-to-modern"
19
+ tokenizer = AutoTokenizer.from_pretrained(PRETRAINED)
20
+ model = EncoderDecoderModel.from_pretrained(PRETRAINED)
21
+ return tokenizer, model
22
+
23
+
24
+ tokenizer, model = load_model()
25
+
26
+
27
+ def inference(text):
28
+ tk_kwargs = dict(
29
+ truncation=True,
30
+ max_length=168,
31
+ padding="max_length",
32
+ return_tensors='pt')
33
+
34
+ inputs = tokenizer([text, ], **tk_kwargs)
35
+ with torch.no_grad():
36
+ return tokenizer.batch_decode(
37
+ model.generate(
38
+ inputs.input_ids,
39
+ attention_mask=inputs.attention_mask,
40
+ num_beams=3,
41
+ max_length=256,
42
+ bos_token_id=101,
43
+ eos_token_id=tokenizer.sep_token_id,
44
+ pad_token_id=tokenizer.pad_token_id,
45
+ ), skip_special_tokens=True)[0].replace(" ","")
46
+
47
+
48
+ @st.cache
49
+ def get_file_df():
50
+ file_df = pd.read_csv("meta.csv")
51
+ return file_df
52
+
53
+
54
+ file_df = get_file_df()
55
+
56
+ col1, col2 = st.columns([.3, 1])
57
+
58
+ col1.markdown("""
59
+ * 朕亲自下厨的[🤗 翻译模型](https://github.com/raynardj/wenyanwen-ancient-translate-to-modern), [⭐️ 训练笔记](https://github.com/raynardj/yuan)
60
+ * 📚 书籍来自 [殆知阁](http://www.daizhige.org/),只为了便于展示翻译,喜欢请访问网站,书籍[github文件链接](https://github.com/garychowcmu/daizhigev20)
61
+ """)
62
+
63
+ USER_ID = st.secrets["USER_ID"]
64
+ SECRET = st.secrets["SECRET"]
65
+
66
+
67
+ @st.cache
68
+ def get_maps():
69
+ file_obj_hash_map = dict(file_df[["filepath", "obj_hash"]].values)
70
+ file_size_map = dict(file_df[["filepath", "fsize"]].values)
71
+ return file_obj_hash_map, file_size_map
72
+
73
+
74
+ file_obj_hash_map, file_size_map = get_maps()
75
+
76
+
77
+ def show_file_size(size: int):
78
+ if size < 1024:
79
+ return f"{size} B"
80
+ elif size < 1024*1024:
81
+ return f"{size//1024} KB"
82
+ else:
83
+ return f"{size/1024//1024} MB"
84
+
85
+
86
+ def fetch_file(path):
87
+ # reading from local path first
88
+ if (Path("data")/path).exists():
89
+ with open(Path("data")/path, "r") as f:
90
+ return f.read()
91
+
92
+ # read from github api
93
+ obj_hash = file_obj_hash_map[path]
94
+ auth = HTTPBasicAuth(USER_ID, SECRET)
95
+ url = f"https://api.github.com/repos/garychowcmu/daizhigev20/git/blobs/{obj_hash}"
96
+ r = requests.get(url, auth=auth)
97
+ if r.status_code == 200:
98
+ data = r.json()
99
+ content = base64.b64decode(data['content']).decode('utf-8')
100
+ return content
101
+ else:
102
+ r.raise_for_status()
103
+
104
+
105
+ def fetch_from_df(sub_paths: str = ""):
106
+ sub_df = file_df.copy()
107
+ for idx, step in enumerate(sub_paths):
108
+ sub_df.query(f"col_{idx} == '{step}'", inplace=True)
109
+ if len(sub_df) == 0:
110
+ return None
111
+ return list(sub_df[f"col_{len(sub_paths)}"].unique())
112
+
113
+
114
+ # root_data = fetch_from_github()
115
+ if 'pathway' in st.session_state:
116
+ pass
117
+ else:
118
+ st.session_state.pathway = []
119
+
120
+ path_text = col1.text("/".join(st.session_state.pathway))
121
+
122
+ def reset_path():
123
+ print("before rooting")
124
+ print("/".join(st.session_state.pathway))
125
+ st.session_state.pathway = []
126
+ path_text.text(st.session_state.pathway)
127
+
128
+ if col1.button("回到根目录"):
129
+ reset_path()
130
+
131
+ def display_tree(sub_list):
132
+ dropdown = col1.selectbox("【选书】", options=sub_list)
133
+ if col1.button(f'【确定{len(st.session_state.pathway)+1}】'):
134
+ st.session_state.pathway.append(dropdown)
135
+ if dropdown.endswith('.txt'):
136
+ filepath = "/".join(st.session_state.pathway)
137
+ file_size = file_size_map[filepath]
138
+ col2.write(
139
+ f"loading file:{filepath},({show_file_size(file_size)})")
140
+
141
+ # if file size is too large, we will not load it
142
+ if file_size > 3*1024*1024:
143
+ urlpath = filepath.replace(".txt",".html")
144
+ dzg = f"http://www.daizhige.org/{urlpath}"
145
+ st.markdown(f"文件太大,[前往殆知阁页面]({dzg}), 或挑挑其他的书吧")
146
+ reset_path()
147
+ return None
148
+ path_text.text(filepath)
149
+ text = fetch_file(filepath)
150
+ # set y scroll markdown
151
+ col2.markdown(f"""```{text}```""", )
152
+ reset_path()
153
+
154
+ else:
155
+ sub_list = fetch_from_df(
156
+ st.session_state.pathway)
157
+ path_text.text("/".join(st.session_state.pathway))
158
+ display_tree(sub_list)
159
+
160
+
161
+ display_tree(fetch_from_df(st.session_state.pathway))
162
+
163
+ cc = st.text_area("【输入文本】", height=150)
164
+
165
+ if st.button("【翻译】"):
166
+ if cc:
167
+ if len(cc)>168:
168
+ st.write(f"句子太长,最多168个字符")
169
+ else:
170
+ st.markdown(f"""```{inference(cc)}```""")
171
+ else:
172
+ st.write("请输入文本")
meta.csv ADDED
The diff for this file is too large to render. See raw diff