eubinecto commited on
Commit
fca50f9
1 Parent(s): 927768a

[#5] literal2idiomatic:d-1-3 done (annotating with special tokens). Some of the data however are erroneous

Browse files
config.yaml CHANGED
@@ -15,7 +15,9 @@ idioms:
15
  ver: d-1-2
16
  description: the set of idioms in the traning set of literal2idiomatic_d-1-2.
17
  literal2idiomatic:
18
- ver: d-1-2
19
- description: PIE data split into train & test set (80 / 20 split). There is no validation set because I don't intend to do any hyperparameter tuning on this thing.
20
  train_ratio: 0.8
21
- seed: 104
 
 
 
15
  ver: d-1-2
16
  description: the set of idioms in the traning set of literal2idiomatic_d-1-2.
17
  literal2idiomatic:
18
+ ver: d-1-3
19
+ description: The idioms are annotated with <idiom> & </idiom>.
20
  train_ratio: 0.8
21
+ seed: 104
22
+ boi_token: <idiom>
23
+ eoi_token: </idiom>
explore/explore_fetch_pie_annotate.py ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ from idiomify.fetchers import fetch_pie
3
+ from preprocess import annotate
4
+
5
+
6
+ def main():
7
+ pie_df = fetch_pie()
8
+ pie_df = pie_df.pipe(annotate, boi_token="<idiom>", eoi_token="</idiom>")
9
+ for _, row in pie_df.iterrows():
10
+ print(row['Idiomatic_Sent'])
11
+
12
+
13
+ if __name__ == '__main__':
14
+ main()
explore/explore_list_index.py ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+
3
+ def main():
4
+ labels = ["O", "O", "B", "O", "I", "I" "O", "I", "O", "O"]
5
+ boi_idx = labels.index("B")
6
+ eoi_idx = -1 * (list(reversed(labels)).index("I") + 1)
7
+ print(boi_idx, eoi_idx)
8
+ print(labels[boi_idx])
9
+ print(labels[eoi_idx])
10
+
11
+
12
+ if __name__ == '__main__':
13
+ main()
idiomify/preprocess.py CHANGED
@@ -17,6 +17,36 @@ def cleanse(df: pd.DataFrame) -> pd.DataFrame:
17
  return df
18
 
19
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
  def stratified_split(df: pd.DataFrame, ratio: float, seed: int) -> Tuple[pd.DataFrame, pd.DataFrame]:
21
  """
22
  stratified-split the given df into two df's.
@@ -29,3 +59,4 @@ def stratified_split(df: pd.DataFrame, ratio: float, seed: int) -> Tuple[pd.Data
29
  test_size=other_size, random_state=seed,
30
  shuffle=True)
31
  return ratio_df, other_df
 
 
17
  return df
18
 
19
 
20
+ def annotate(df: pd.DataFrame, boi_token: str, eoi_token: str) -> pd.DataFrame:
21
+ """
22
+ e.g.
23
+ given a row like this:
24
+ Idiom keep an eye on
25
+ Sense keep a watch on something or someone closely
26
+ Idiomatic_Sent He had put on a lot of weight lately , so he started keeping an eye on what he ate .
27
+ Literal_Sent He had put on a lot of weight lately , so he started to watch what he ate .
28
+ Idiomatic_Label O O O O O O O O O O O O O B I I O O O O O
29
+ Literal_Label O O O O O O O O O O O O O B I O O O O
30
+
31
+ use Idiomatic_Label to replace Idiomatic_Sent with:
32
+ He had put on a lot of weight lately , so he started <idiom> keeping an eye on </idiom> what he ate .
33
+ """
34
+ for idx, row in df.iterrows():
35
+ tokens = row['Idiomatic_Sent'].split(" ")
36
+ labels = row["Idiomatic_Label"].split(" ")
37
+ if "B" in labels:
38
+ boi_idx = labels.index("B")
39
+ if "I" in labels:
40
+ eoi_idx = -1 * (list(reversed(labels)).index("I") + 1)
41
+ tokens[boi_idx] = f"{boi_token} {tokens[boi_idx]}"
42
+ tokens[eoi_idx] = f"{tokens[eoi_idx]} {eoi_token}"
43
+ else:
44
+ tokens[boi_idx] = f"{boi_token} {tokens[boi_idx]} {eoi_token}"
45
+ row['Idiomatic_Sent'] = " ".join(tokens)
46
+
47
+ return df
48
+
49
+
50
  def stratified_split(df: pd.DataFrame, ratio: float, seed: int) -> Tuple[pd.DataFrame, pd.DataFrame]:
51
  """
52
  stratified-split the given df into two df's.
 
59
  test_size=other_size, random_state=seed,
60
  shuffle=True)
61
  return ratio_df, other_df
62
+
main_upload_literal2idiomatic.py CHANGED
@@ -4,7 +4,7 @@ literal2idiomatic ver: d-1-2
4
  import os
5
  from idiomify.paths import ROOT_DIR
6
  from idiomify.fetchers import fetch_pie, fetch_config
7
- from idiomify.preprocess import upsample, cleanse, stratified_split
8
  import wandb
9
 
10
 
@@ -15,6 +15,7 @@ def main():
15
  config = fetch_config()['literal2idiomatic']
16
  train_df, test_df = pie_df.pipe(cleanse)\
17
  .pipe(upsample, seed=config['seed'])\
 
18
  .pipe(stratified_split, ratio=config['train_ratio'], seed=config['seed'])
19
  # why don't you just "select" the columns? yeah, stop using csv library. just select them.
20
  train_df = train_df[["Idiom", "Literal_Sent", "Idiomatic_Sent"]]
 
4
  import os
5
  from idiomify.paths import ROOT_DIR
6
  from idiomify.fetchers import fetch_pie, fetch_config
7
+ from idiomify.preprocess import upsample, cleanse, stratified_split, annotate
8
  import wandb
9
 
10
 
 
15
  config = fetch_config()['literal2idiomatic']
16
  train_df, test_df = pie_df.pipe(cleanse)\
17
  .pipe(upsample, seed=config['seed'])\
18
+ .pipe(annotate, boi_token=config['boi_token'], eoi_token=config['eoi_token'])\
19
  .pipe(stratified_split, ratio=config['train_ratio'], seed=config['seed'])
20
  # why don't you just "select" the columns? yeah, stop using csv library. just select them.
21
  train_df = train_df[["Idiom", "Literal_Sent", "Idiomatic_Sent"]]