Spaces:
Runtime error
Runtime error
Commit
•
bbb171e
1
Parent(s):
7082f08
feat: hashtag segmentation
Browse files- app.py +130 -0
- requirements.txt +3 -0
app.py
ADDED
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import gradio as gr
|
2 |
+
from hashformers import TransformerWordSegmenter as WordSegmenter
|
3 |
+
import pandas as pd
|
4 |
+
|
5 |
+
article_string = "Author: <a href=\"https://huggingface.co/ruanchaves\">Ruan Chaves Rodrigues</a>. Read more about the <a href=\"https://github.com/ruanchaves/hashformers\">Hashformers library</a>."
|
6 |
+
|
7 |
+
app_title = "Hashtag segmentation"
|
8 |
+
|
9 |
+
app_description = """
|
10 |
+
Hashtag segmentation is the task of automatically adding spaces between the words on a hashtag.
|
11 |
+
This app uses the <a href=\"https://github.com/ruanchaves/hashformers\">Hashformers library</a> to suggest segmentations for hashtags.
|
12 |
+
|
13 |
+
Enter a hashtag or pick one from the examples below. The app will suggest the best segmentation for the hashtag.
|
14 |
+
"""
|
15 |
+
|
16 |
+
app_examples = [
|
17 |
+
["#helloworld"]
|
18 |
+
]
|
19 |
+
|
20 |
+
output_json_component_description = {"": ""}
|
21 |
+
|
22 |
+
model_dict = {
|
23 |
+
"english": WordSegmenter(
|
24 |
+
segmenter_model_name_or_path="gpt2",
|
25 |
+
reranker_model_name_or_path="bert-base-uncased",
|
26 |
+
segmenter_device="cpu",
|
27 |
+
),
|
28 |
+
"english (fast)": WordSegmenter(
|
29 |
+
segmenter_model_name_or_path="distilgpt2",
|
30 |
+
reranker_model_name_or_path="distilbert-base-uncased",
|
31 |
+
segmenter_device="cpu",
|
32 |
+
),
|
33 |
+
"spanish": WordSegmenter(
|
34 |
+
segmenter_model_name_or_path="mrm8488/spanish-gpt2",
|
35 |
+
reranker_model_name_or_path="dccuchile/bert-base-spanish-wwm-cased",
|
36 |
+
segmenter_device="cpu",
|
37 |
+
),
|
38 |
+
"portuguese": WordSegmenter(
|
39 |
+
segmenter_model_name_or_path="pierreguillou/gpt2-small-portuguese",
|
40 |
+
reranker_model_name_or_path="neuralmind/bert-base-portuguese-cased",
|
41 |
+
segmenter_device="cpu",
|
42 |
+
),
|
43 |
+
"german": WordSegmenter(
|
44 |
+
segmenter_model_name_or_path="dbmdz/german-gpt2",
|
45 |
+
reranker_model_name_or_path="bert-base-german-cased",
|
46 |
+
segmenter_device="cpu",
|
47 |
+
),
|
48 |
+
}
|
49 |
+
|
50 |
+
language_list = list(model_dict.keys())
|
51 |
+
|
52 |
+
def format_dataframe(df):
|
53 |
+
if not isinstance(df, pd.DataFrame):
|
54 |
+
return df
|
55 |
+
df = df[["segmentation", "score"]]
|
56 |
+
df["score"] = df["score"].apply(lambda x: 1/x)
|
57 |
+
df["score"] = df["score"].apply(lambda x: round(x, 4))
|
58 |
+
return df
|
59 |
+
|
60 |
+
def convert_to_score_dict(df):
|
61 |
+
if not isinstance(df, pd.DataFrame):
|
62 |
+
return {}
|
63 |
+
df = df[["segmentation", "score"]]
|
64 |
+
return df.set_index("segmentation").T.to_dict("records")[0]
|
65 |
+
|
66 |
+
def get_candidates_df(candidates, segmenter_score_dict, reranker_score_dict ):
|
67 |
+
candidates_df = []
|
68 |
+
for candidate in candidates:
|
69 |
+
candidates_df.append(
|
70 |
+
{
|
71 |
+
"segmentation": candidate,
|
72 |
+
"segmenter score": segmenter_score_dict.get(candidate, 0),
|
73 |
+
"reranker score": reranker_score_dict.get(candidate, 0),
|
74 |
+
})
|
75 |
+
candidates_df = pd.DataFrame(candidates_df)
|
76 |
+
return candidates_df
|
77 |
+
|
78 |
+
def parse_candidates(candidates):
|
79 |
+
if not candidates:
|
80 |
+
return []
|
81 |
+
candidates = candidates.split(",")
|
82 |
+
candidates = [c.strip() for c in candidates]
|
83 |
+
return candidates
|
84 |
+
|
85 |
+
def predict(s1, language, candidates, use_reranker, topk, steps):
|
86 |
+
hashtag_list = [s1]
|
87 |
+
segmentation = model_dict[language].segment(hashtag_list, use_reranker=use_reranker, return_ranks=True, topk=topk, steps=steps)
|
88 |
+
segmenter_df = format_dataframe(segmentation.segmenter_rank)
|
89 |
+
reranker_df = format_dataframe(segmentation.reranker_rank)
|
90 |
+
|
91 |
+
|
92 |
+
top_segmentation = segmentation.output[0]
|
93 |
+
segmenter_score_dict = convert_to_score_dict(segmenter_df)
|
94 |
+
reranker_score_dict = convert_to_score_dict(reranker_df)
|
95 |
+
top_segmentation_df = get_candidates_df([top_segmentation], segmenter_score_dict, reranker_score_dict)
|
96 |
+
|
97 |
+
candidates_list = parse_candidates(candidates)
|
98 |
+
|
99 |
+
candidates_df = get_candidates_df(candidates_list, segmenter_score_dict, reranker_score_dict)
|
100 |
+
output_df = pd.concat([top_segmentation_df, candidates_df], axis=0)
|
101 |
+
|
102 |
+
if use_reranker:
|
103 |
+
output_df = output_df.sort_values(by="reranker score", ascending=False)
|
104 |
+
else:
|
105 |
+
output_df = output_df.sort_values(by="segmenter score", ascending=False)
|
106 |
+
|
107 |
+
output_df = output_df.drop_duplicates(subset="segmentation", keep="first")
|
108 |
+
|
109 |
+
return top_segmentation, output_df
|
110 |
+
|
111 |
+
|
112 |
+
inputs = [
|
113 |
+
gr.Textbox(label="Hashtag"),
|
114 |
+
gr.Dropdown(language_list, label="Language", value="english (fast)"),
|
115 |
+
gr.Textbox(label="Candidate segmentations"),
|
116 |
+
gr.Checkbox(label="Use reranker", value=True),
|
117 |
+
gr.Slider(0, 100, value=20, label="Advanced setting - Beamsearch top-k"),
|
118 |
+
gr.Slider(0, 100, value=13, label="Advanced setting - Beamsearch steps")
|
119 |
+
]
|
120 |
+
|
121 |
+
outputs = [
|
122 |
+
gr.Textbox(label="Suggested segmentation"),
|
123 |
+
gr.DataFrame(label="Scores"),
|
124 |
+
]
|
125 |
+
|
126 |
+
|
127 |
+
gr.Interface(fn=predict, inputs=inputs, outputs=outputs, title=app_title,
|
128 |
+
description=app_description,
|
129 |
+
examples=app_examples,
|
130 |
+
article = article_string).launch()
|
requirements.txt
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
torch
|
2 |
+
gradio
|
3 |
+
hashformers
|