bkhmsi commited on
Commit
3034f4c
โ€ข
1 Parent(s): e5cfa50

updated with paper link

Browse files
Files changed (2) hide show
  1. PartialDD.png +0 -0
  2. app.py +27 -27
PartialDD.png CHANGED
app.py CHANGED
@@ -59,11 +59,11 @@ with gr.Blocks(theme=gr.themes.Default(text_size="lg")) as demo:
59
 
60
  gr.Markdown(
61
  """
62
- <img src='https://huggingface.co/spaces/bkhmsi/Partial-Arabic-Diacritization/resolve/main/PartialDD.png' style='float:right; margin: 0 0 10px 10px;'/>
63
 
64
  <h1> Partial Diacritization: A Context-Contrastive Inference Approach </h1>
65
  <h2> Authors: Muhammad ElNokrashy, Badr AlKhamissi </h2>
66
- <h3> Paper Link: TBD </h3>
67
  <details>
68
  <summary>Abstract</summary>
69
  <p>Diacritization plays a pivotal role in improving readability and disambiguating the meaning of Arabic texts. Efforts have so far focused on marking every eligible character (Full Diacritization). Comparatively overlooked, Partial Diacritzation (PD) is the selection of a subset of characters to be marked to aid comprehension where needed.Research has indicated that excessive diacritic marks can hinder skilled readers---reducing reading speed and accuracy. We conduct a behavioral experiment and show that partially marked text is often easier to read than fully marked text, and sometimes easier than plain text. In this light, we introduce Context-Contrastive Partial Diacritization (CCPD)---a novel approach to PD which integrates seamlessly with existing Arabic diacritization systems. CCPD processes each word twice, once with context and once without, and diacritizes only the characters with disparities between the two inferences. Further, we introduce novel indicators for measuring partial diacritization quality {SR, PDER, HDER, ERE}, essential for establishing this as a machine learning task. Lastly, we introduce TD2, a Transformer-variant of an established model which offers a markedly different performance profile on our proposed indicators compared to all other known systems.</p>
@@ -77,9 +77,12 @@ with gr.Blocks(theme=gr.themes.Default(text_size="lg")) as demo:
77
  value=current_model_name
78
  )
79
 
80
- with gr.Tab(label="Full Diacritization"):
81
-
82
- full_input_txt = gr.Textbox(
 
 
 
83
  placeholder="ุงูƒุชุจ ู‡ู†ุง",
84
  lines=5,
85
  label="Input",
@@ -88,7 +91,7 @@ with gr.Blocks(theme=gr.themes.Default(text_size="lg")) as demo:
88
  text_align='right',
89
  )
90
 
91
- full_output_txt = gr.Textbox(
92
  lines=5,
93
  label="Output",
94
  type='text',
@@ -97,25 +100,24 @@ with gr.Blocks(theme=gr.themes.Default(text_size="lg")) as demo:
97
  show_copy_button=True,
98
  )
99
 
100
- full_btn = gr.Button(value="Shakkel")
101
- full_btn.click(diacritze_full, inputs=[full_input_txt, model_choice], outputs=[full_output_txt], queue=True)
102
 
103
  gr.Examples(
104
  examples=[
105
- ["ูˆู„ูˆ ุญู…ู„ ู…ู† ู…ุฌู„ุณ ุงู„ุฎูŠุงุฑ ุŒ ูˆู„ู… ูŠู…ู†ุน ู…ู† ุงู„ูƒู„ุงู…", "TD2"],
 
 
106
  ],
107
- inputs=[full_input_txt, model_choice],
108
- outputs=full_output_txt,
109
- fn=diacritze_full,
110
  cache_examples=True,
111
  )
112
 
113
- with gr.Tab(label="Partial Diacritization") as partial_settings:
114
- with gr.Row():
115
- masking_mode = gr.Radio(choices=["Hard", "Soft"], value="Hard", label="Masking Mode")
116
- threshold_slider = gr.Slider(label="Soft Masking Threshold", minimum=0, maximum=1, value=0.1)
117
-
118
- partial_input_txt = gr.Textbox(
119
  placeholder="ุงูƒุชุจ ู‡ู†ุง",
120
  lines=5,
121
  label="Input",
@@ -124,7 +126,7 @@ with gr.Blocks(theme=gr.themes.Default(text_size="lg")) as demo:
124
  text_align='right',
125
  )
126
 
127
- partial_output_txt = gr.Textbox(
128
  lines=5,
129
  label="Output",
130
  type='text',
@@ -133,18 +135,16 @@ with gr.Blocks(theme=gr.themes.Default(text_size="lg")) as demo:
133
  show_copy_button=True,
134
  )
135
 
136
- partial_btn = gr.Button(value="Shakkel")
137
- partial_btn.click(diacritze_partial, inputs=[partial_input_txt, masking_mode, threshold_slider, model_choice], outputs=[partial_output_txt], queue=True)
138
 
139
  gr.Examples(
140
  examples=[
141
- ["ูˆู„ูˆ ุญู…ู„ ู…ู† ู…ุฌู„ุณ ุงู„ุฎูŠุงุฑ ุŒ ูˆู„ู… ูŠู…ู†ุน ู…ู† ุงู„ูƒู„ุงู…", "Hard", 0, "TD2"],
142
- ["ูˆู„ูˆ ุญู…ู„ ู…ู† ู…ุฌู„ุณ ุงู„ุฎูŠุงุฑ ุŒ ูˆู„ู… ูŠู…ู†ุน ู…ู† ุงู„ูƒู„ุงู…", "Soft", 0.1, "TD2"],
143
- ["ูˆู„ูˆ ุญู…ู„ ู…ู† ู…ุฌู„ุณ ุงู„ุฎูŠุงุฑ ุŒ ูˆู„ู… ูŠู…ู†ุน ู…ู† ุงู„ูƒู„ุงู…", "Soft", 0.01, "TD2"],
144
  ],
145
- inputs=[partial_input_txt, masking_mode, threshold_slider, model_choice],
146
- outputs=partial_output_txt,
147
- fn=diacritze_partial,
148
  cache_examples=True,
149
  )
150
 
 
59
 
60
  gr.Markdown(
61
  """
62
+ <img src='https://huggingface.co/spaces/bkhmsi/Partial-Arabic-Diacritization/resolve/main/PartialDD.png' style='float:right; margin: 0 0 10px 10px; width: 20%'/>
63
 
64
  <h1> Partial Diacritization: A Context-Contrastive Inference Approach </h1>
65
  <h2> Authors: Muhammad ElNokrashy, Badr AlKhamissi </h2>
66
+ <h3> Paper: <a href="https://arxiv.org/abs/2401.08919"> https://arxiv.org/abs/2401.08919 </a> </h3>
67
  <details>
68
  <summary>Abstract</summary>
69
  <p>Diacritization plays a pivotal role in improving readability and disambiguating the meaning of Arabic texts. Efforts have so far focused on marking every eligible character (Full Diacritization). Comparatively overlooked, Partial Diacritzation (PD) is the selection of a subset of characters to be marked to aid comprehension where needed.Research has indicated that excessive diacritic marks can hinder skilled readers---reducing reading speed and accuracy. We conduct a behavioral experiment and show that partially marked text is often easier to read than fully marked text, and sometimes easier than plain text. In this light, we introduce Context-Contrastive Partial Diacritization (CCPD)---a novel approach to PD which integrates seamlessly with existing Arabic diacritization systems. CCPD processes each word twice, once with context and once without, and diacritizes only the characters with disparities between the two inferences. Further, we introduce novel indicators for measuring partial diacritization quality {SR, PDER, HDER, ERE}, essential for establishing this as a machine learning task. Lastly, we introduce TD2, a Transformer-variant of an established model which offers a markedly different performance profile on our proposed indicators compared to all other known systems.</p>
 
77
  value=current_model_name
78
  )
79
 
80
+ with gr.Tab(label="Partial Diacritization") as partial_settings:
81
+ with gr.Row():
82
+ masking_mode = gr.Radio(choices=["Hard", "Soft"], value="Hard", label="Masking Mode")
83
+ threshold_slider = gr.Slider(label="Soft Masking Threshold", minimum=0, maximum=1, value=0.1)
84
+
85
+ partial_input_txt = gr.Textbox(
86
  placeholder="ุงูƒุชุจ ู‡ู†ุง",
87
  lines=5,
88
  label="Input",
 
91
  text_align='right',
92
  )
93
 
94
+ partial_output_txt = gr.Textbox(
95
  lines=5,
96
  label="Output",
97
  type='text',
 
100
  show_copy_button=True,
101
  )
102
 
103
+ partial_btn = gr.Button(value="Shakkel")
104
+ partial_btn.click(diacritze_partial, inputs=[partial_input_txt, masking_mode, threshold_slider, model_choice], outputs=[partial_output_txt], queue=True)
105
 
106
  gr.Examples(
107
  examples=[
108
+ ["ูˆู„ูˆ ุญู…ู„ ู…ู† ู…ุฌู„ุณ ุงู„ุฎูŠุงุฑ ุŒ ูˆู„ู… ูŠู…ู†ุน ู…ู† ุงู„ูƒู„ุงู…", "Hard", 0, "TD2"],
109
+ ["ูˆู„ูˆ ุญู…ู„ ู…ู† ู…ุฌู„ุณ ุงู„ุฎูŠุงุฑ ุŒ ูˆู„ู… ูŠู…ู†ุน ู…ู† ุงู„ูƒู„ุงู…", "Soft", 0.1, "TD2"],
110
+ ["ูˆู„ูˆ ุญู…ู„ ู…ู† ู…ุฌู„ุณ ุงู„ุฎูŠุงุฑ ุŒ ูˆู„ู… ูŠู…ู†ุน ู…ู† ุงู„ูƒู„ุงู…", "Soft", 0.01, "TD2"],
111
  ],
112
+ inputs=[partial_input_txt, masking_mode, threshold_slider, model_choice],
113
+ outputs=partial_output_txt,
114
+ fn=diacritze_partial,
115
  cache_examples=True,
116
  )
117
 
118
+ with gr.Tab(label="Full Diacritization"):
119
+
120
+ full_input_txt = gr.Textbox(
 
 
 
121
  placeholder="ุงูƒุชุจ ู‡ู†ุง",
122
  lines=5,
123
  label="Input",
 
126
  text_align='right',
127
  )
128
 
129
+ full_output_txt = gr.Textbox(
130
  lines=5,
131
  label="Output",
132
  type='text',
 
135
  show_copy_button=True,
136
  )
137
 
138
+ full_btn = gr.Button(value="Shakkel")
139
+ full_btn.click(diacritze_full, inputs=[full_input_txt, model_choice], outputs=[full_output_txt], queue=True)
140
 
141
  gr.Examples(
142
  examples=[
143
+ ["ูˆู„ูˆ ุญู…ู„ ู…ู† ู…ุฌู„ุณ ุงู„ุฎูŠุงุฑ ุŒ ูˆู„ู… ูŠู…ู†ุน ู…ู† ุงู„ูƒู„ุงู…", "TD2"],
 
 
144
  ],
145
+ inputs=[full_input_txt, model_choice],
146
+ outputs=full_output_txt,
147
+ fn=diacritze_full,
148
  cache_examples=True,
149
  )
150