boris commited on
Commit
d054d1b
1 Parent(s): 7b8c2cb

feat: simplify fix_html

Browse files
Files changed (1) hide show
  1. dalle_mini/text.py +2 -17
dalle_mini/text.py CHANGED
@@ -98,23 +98,8 @@ def replace_person_token(t):
98
 
99
 
100
  def fix_html(t):
101
- "Adapted from fastai"
102
- t = (
103
- t.replace("#39;", "'")
104
- .replace("&", "&")
105
- .replace("amp;", "&")
106
- .replace("#146;", "'")
107
- .replace("nbsp;", " ")
108
- .replace("#36;", "$")
109
- .replace("\\n", "\n")
110
- .replace("quot;", "'")
111
- .replace("<br />", "\n")
112
- .replace('\\"', '"')
113
- .replace("<unk>", " ")
114
- .replace(" @.@ ", ".")
115
- .replace(" @-@ ", "-")
116
- )
117
- return html.unescape(t)
118
 
119
 
120
  def replace_punctuation_with_commas(t):
 
98
 
99
 
100
  def fix_html(t):
101
+ # from OpenAI CLIP
102
+ return html.unescape(html.unescape(t)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
 
104
 
105
  def replace_punctuation_with_commas(t):