[#5] idioms:d-1-3 is done. Added multiple senses of the same idiom as a list

config.yaml

@@ -12,8 +12,8 @@ idiomifier:
 
 # for building & uploading datasets or tokenizer
 idioms:
-  ver: d-1-2
-  description: the set of idioms in the traning set of literal2idiomatic_d-1-2.
+  ver: d-1-3
+  description: the set of idioms in the traning set of literal2idiomatic_d-1-3. Definitions of them are added as well.
 literal2idiomatic:
   ver: d-1-3
   description: The idioms are annotated with <idiom> & </idiom>.

explore/explore_fetch_idioms.py

@@ -2,7 +2,7 @@ from idiomify.fetchers import fetch_idioms
 
 
 def main():
-    print(fetch_idioms("d-1-2"))
+    print(fetch_idioms("d-1-3"))
 
 
 if __name__ == '__main__':

explore/explore_upload_idioms_groupby.py

@@ -0,0 +1,22 @@
+from idiomify.fetchers import fetch_literal2idiomatic, fetch_config
+
+
+def main():
+    config = fetch_config()['literal2idiomatic']
+    train_df, _ = fetch_literal2idiomatic(config['ver'])
+    idioms_df = train_df[['Idiom', "Sense"]]
+    idioms_df = idioms_df.groupby('Idiom').agg({'Sense': lambda x: list(set(x))})
+    print(idioms_df.head(5))
+    for idx, row in idioms_df.iterrows():
+        print(row['Sense'])
+
+"""
+['to arrange something in a manner that either someone will gain a wrong disadvantage or a person would get an unfair advantage']
+['Used in general to refer an experience or talent or ability or position, which would be useful or beneficial for a person, his life and his future.']
+['to be very easy to see or notice']
+[' to reach a logical conclusion']
+['to start doing something over from the beginning']
+"""
+
+if __name__ == '__main__':
+    main()

idiomify/fetchers.py

@@ -17,7 +17,7 @@ def fetch_pie() -> pd.DataFrame:
 
 
 # --- from wandb --- #
-def fetch_idioms(ver: str, run: Run = None) -> List[str]:
+def fetch_idioms(ver: str, run: Run = None) -> pd.DataFrame:
     """
     why do you need this? -> you need this to have access to the idiom embeddings.
     """
@@ -28,9 +28,8 @@ def fetch_idioms(ver: str, run: Run = None) -> List[str]:
     else:
         artifact = wandb.Api().artifact(f"eubinecto/idiomify/idioms:{ver}", type="dataset")
     artifact_dir = artifact.download(root=idioms_dir(ver))
-    txt_path = path.join(artifact_dir, "all.txt")
-    with open(txt_path, 'r') as fh:
-        return [line.strip() for line in fh]
+    tsv_path = path.join(artifact_dir, "all.tsv")
+    return pd.read_csv(tsv_path, sep="\t")
 
 
 def fetch_literal2idiomatic(ver: str, run: Run = None) -> Tuple[pd.DataFrame, pd.DataFrame]:

main_upload_idioms.py

@@ -11,22 +11,20 @@ from idiomify.paths import ROOT_DIR
 def main():
     config = fetch_config()['idioms']
     train_df, _ = fetch_literal2idiomatic(config['ver'])
-    idioms = train_df['Idiom'].tolist()
-    idioms = list(set(idioms))
+    idioms_df = train_df[['Idiom', "Sense"]]
+    idioms_df = idioms_df.groupby('Idiom').agg({'Sense': lambda x: list(set(x))})
 
     with wandb.init(entity="eubinecto", project="idiomify") as run:
         # the paths to write datasets in
-        txt_path = ROOT_DIR / "all.txt"
-        with open(txt_path, 'w') as fh:
-            for idiom in idioms:
-                fh.write(idiom + "\n")
+        tsv_path = ROOT_DIR / "all.tsv"
+        idioms_df.to_csv(tsv_path, sep="\t")
         artifact = wandb.Artifact(name="idioms", type="dataset", description=config['description'],
                                   metadata=config)
-        artifact.add_file(txt_path)
+        artifact.add_file(tsv_path)
         # then, we just log them here.
         run.log_artifact(artifact, aliases=["latest", config['ver']])
         # don't forget to remove them
-        os.remove(txt_path)
+        os.remove(tsv_path)
 
 
 if __name__ == '__main__':

main_upload_literal2idiomatic.py

@@ -18,8 +18,8 @@ def main():
                               .pipe(annotate, boi_token=config['boi_token'], eoi_token=config['eoi_token'])\
                               .pipe(stratified_split, ratio=config['train_ratio'], seed=config['seed'])
     # why don't you just "select"  the columns? yeah, stop using csv library. just select them.
-    train_df = train_df[["Idiom", "Literal_Sent", "Idiomatic_Sent"]]
-    test_df = test_df[["Idiom", "Literal_Sent", "Idiomatic_Sent"]]
+    train_df = train_df[["Idiom", "Sense", "Literal_Sent", "Idiomatic_Sent"]]
+    test_df = test_df[["Idiom", "Sense", "Literal_Sent", "Idiomatic_Sent"]]
     dfs = (train_df, test_df)
     with wandb.init(entity="eubinecto", project="idiomify") as run:
         # the paths to write datasets in