tjtanaa commited on
Commit
dcd4eae
1 Parent(s): c7e10bf

Add tokenizer_config.json download

Browse files
Files changed (1) hide show
  1. app.py +27 -8
app.py CHANGED
@@ -45,6 +45,12 @@ if 'uuid' not in st.session_state:
45
  if 'successful_template' not in st.session_state:
46
  st.session_state['successful_template'] = ''
47
 
 
 
 
 
 
 
48
  if not os.path.exists("./tmp"):
49
  os.makedirs("./tmp")
50
 
@@ -97,7 +103,7 @@ if st.session_state['tokenizer_json'] is not None:
97
  {"role": "user", "content": "I'd like to show off how chat templating works!"},
98
  ]
99
  st.markdown("### Example Conversation")
100
- st.json(json.dumps(chat, indent=4))
101
 
102
  prompt_template_col, prompt_template_output_col = st.columns(spec=[0.3, 0.7])
103
 
@@ -141,22 +147,35 @@ if st.session_state['tokenizer_json'] is not None:
141
  generated_prompt_wo_add_generation_prompt = st.session_state['tokenizer'].apply_chat_template(chat, tokenize=False, add_generation_prompt= False)
142
  generated_prompt_w_add_generation_prompt = st.session_state['tokenizer'].apply_chat_template(chat, tokenize=False, add_generation_prompt= True)
143
 
 
 
 
 
144
  st.text_area(
145
- "Generate Prompt with `add_generation_prompt=False`", value=generated_prompt_wo_add_generation_prompt,
146
  height=300, placeholder=None, disabled=True, label_visibility="visible", key="generated_prompt_wo_add_generation_prompt")
147
 
148
  st.text_area(
149
- "Generate Prompt with `add_generation_prompt=True`", value=generated_prompt_w_add_generation_prompt,
150
  height=300, placeholder=None, disabled=True, label_visibility="visible", key="generated_prompt_w_add_generation_prompt")
151
-
152
- st.session_state['successful_template'] = copy.deepcopy(st.session_state['input_jinja_template'])
153
-
154
- if len(st.session_state['successful_template']) > 0:
155
  access_token_no_cache = st.text_input("HuggingFace Access Token API with Write Access", type="password", key="access_token_no_cache")
156
  commit_message_text_input = st.text_input("Commit Message", key="commit_message_text_input")
157
  to_private_checkbox = st.checkbox("To Private Repo", key="to_private_checkbox")
158
- push_to_hub_button = st.button("Push to Hub", key="push_to_hub_button")
159
  create_pr_checkbox = st.checkbox("Create PR (For Contribution 🤗)", key="create_pr_checkbox")
 
 
 
 
 
 
 
 
 
 
 
 
 
160
  if push_to_hub_button:
161
  with open(f"./tmp/{st.session_state['uuid']}/tmp_chat_template.json", "w") as fp:
162
  fp.write(st.session_state['successful_template'])
 
45
  if 'successful_template' not in st.session_state:
46
  st.session_state['successful_template'] = ''
47
 
48
+ if 'generated_prompt_w_add_generation_prompt' not in st.session_state:
49
+ st.session_state['generated_prompt_w_add_generation_prompt'] = ''
50
+
51
+ if 'generated_prompt_wo_add_generation_prompt' not in st.session_state:
52
+ st.session_state['generated_prompt_wo_add_generation_prompt'] = ''
53
+
54
  if not os.path.exists("./tmp"):
55
  os.makedirs("./tmp")
56
 
 
103
  {"role": "user", "content": "I'd like to show off how chat templating works!"},
104
  ]
105
  st.markdown("### Example Conversation")
106
+ st.json(json.dumps(chat, indent=4), expanded=False)
107
 
108
  prompt_template_col, prompt_template_output_col = st.columns(spec=[0.3, 0.7])
109
 
 
147
  generated_prompt_wo_add_generation_prompt = st.session_state['tokenizer'].apply_chat_template(chat, tokenize=False, add_generation_prompt= False)
148
  generated_prompt_w_add_generation_prompt = st.session_state['tokenizer'].apply_chat_template(chat, tokenize=False, add_generation_prompt= True)
149
 
150
+ st.session_state['successful_template'] = copy.deepcopy(st.session_state['input_jinja_template'])
151
+
152
+
153
+ if len(st.session_state['successful_template']) > 0:
154
  st.text_area(
155
+ "Generate Prompt with `add_generation_prompt=False`", value=st.session_state['generated_prompt_wo_add_generation_prompt'],
156
  height=300, placeholder=None, disabled=True, label_visibility="visible", key="generated_prompt_wo_add_generation_prompt")
157
 
158
  st.text_area(
159
+ "Generate Prompt with `add_generation_prompt=True`", value=st.session_state['generated_prompt_w_add_generation_prompt'],
160
  height=300, placeholder=None, disabled=True, label_visibility="visible", key="generated_prompt_w_add_generation_prompt")
161
+
 
 
 
162
  access_token_no_cache = st.text_input("HuggingFace Access Token API with Write Access", type="password", key="access_token_no_cache")
163
  commit_message_text_input = st.text_input("Commit Message", key="commit_message_text_input")
164
  to_private_checkbox = st.checkbox("To Private Repo", key="to_private_checkbox")
 
165
  create_pr_checkbox = st.checkbox("Create PR (For Contribution 🤗)", key="create_pr_checkbox")
166
+ push_to_hub_button = st.button("Push to Hub", key="push_to_hub_button", use_container_width=True)
167
+ st.session_state['tokenizer'].save_pretrained(f"./tmp/{st.session_state['uuid']}_{hf_model_repo_name}")
168
+ with open(f"./tmp/{st.session_state['uuid']}_{hf_model_repo_name}/tokenizer_config.json", "r") as f:
169
+
170
+ tokenizer_config_content = json.loads(f.read())
171
+
172
+ st.download_button(
173
+ label="Download tokenizer_config.json",
174
+ data=json.dumps(tokenizer_config_content, indent=4),
175
+ file_name='tokenizer_config.json',
176
+ mime='application/json',
177
+ use_container_width=True
178
+ )
179
  if push_to_hub_button:
180
  with open(f"./tmp/{st.session_state['uuid']}/tmp_chat_template.json", "w") as fp:
181
  fp.write(st.session_state['successful_template'])