Repository: adansons/base (language: python)

Issue:
can't specify table joining rule by `base import --external-file`
# Motivation
When I execute `base import --external-file`, 
I want to modify the estimated join rule, but I can't now.
# Description

# Additional context (optional)



Discussion / hints:

Fix patch:
diff --git a/README.md b/README.md
index 3f06bd7..a2e5d48 100644
--- a/README.md
+++ b/README.md
@@ -137,7 +137,7 @@ after downloading, you can see data files in ~/dataset/mnist.
 
 ### Step 1. create a new project
 
-create mnist project with [base new <project>](https://vast-bus-1c4.notion.site/Command-Reference-20fe913cdf2541cc90812f69de0ae083#70af41e65b51453f9b6807dc29354002) command.
+create mnist project with [base new <project>](/docs/CLI.md#new) command.
 
 ```bash
 base new mnist
@@ -263,7 +263,7 @@ You can use these operators below in the query option.
   not in : not inner list of Values
 ```
 
-(check [search docs](https://vast-bus-1c4.notion.site/Command-Reference-20fe913cdf2541cc90812f69de0ae083#698aab257e884e6a82cf470ec7f459dc) for more information).
+(check [search docs](/docs/CLI.md#search) for more information).
 
 ```bash
 base search mnist --conditions "train" --query "label in ['1','2','3']"
@@ -288,7 +288,7 @@ If you want to use the 'OR search' with the `--query` command, please use our Py
 
 ### Step 5. filter and export dataset with Python SDK
 
-in python script, you can filter and export datasets easily and simply with `Project class` and `Files class`. (see [SDK docs](https://vast-bus-1c4.notion.site/Python-Reference-e4b62aee4eee47caa96b7f436d6658cd))
+in python script, you can filter and export datasets easily and simply with `Project class` and `Files class`. (see [SDK docs](/docs/SDK.md#project-class))
 
 (If you don't have the packages below, please install them by using `pip`)
 ```bash
@@ -365,8 +365,8 @@ eval_dataset = Dataset(eval_files, target_key="label", transform=preprocess_func
 
 ### 4.1 Command Reference
 
-[Command Reference](https://vast-bus-1c4.notion.site/Command-Reference-20fe913cdf2541cc90812f69de0ae083)
+[Command Reference](/docs/CLI.md)
 
 ### 4.2 Python Reference
 
-[Python Reference](https://vast-bus-1c4.notion.site/Python-Reference-e4b62aee4eee47caa96b7f436d6658cd)
+[Python Reference](/docs/SDK.md)
diff --git a/base/cli.py b/base/cli.py
index 872a82b..8eb26b5 100644
--- a/base/cli.py
+++ b/base/cli.py
@@ -297,6 +297,7 @@ def show_project_detail(project, user_id, member_list):
     help="path for external meta-data file",
     required=False,
     default=None,
+    multiple=True,
 )
 @click.option(
     "-d",
@@ -326,6 +327,29 @@ def show_project_detail(project, user_id, member_list):
     default=None,
     multiple=True,
 )
+@click.option(
+    "--extract",
+    type=str,
+    help="flag for extract external file",
+    is_flag=True,
+    default=False,
+)
+@click.option(
+    "--estimate-rule",
+    type=str,
+    help="flag for estimate join rule",
+    is_flag=True,
+    default=False,
+)
+@click.option(
+    "--join-rule",
+    type=str,
+    help="file path for defining the join rule",
+    required=False,
+    default=None,
+)
+@click.option("--export", type=str, help="export file type", required=False)
+@click.option("--output", type=str, help="output file path", required=False)
 @click.option("--auto-approve", is_flag=True)
 @base_config
 def import_data(
@@ -337,6 +361,11 @@ def import_data(
     parse,
     additional,
     auto_approve,
+    extract,
+    estimate_rule,
+    join_rule,
+    export,
+    output,
     user_id,
 ):
     """
@@ -374,7 +403,15 @@ def import_data(
             )
         else:
             import_metafile(
-                project, path, additional, auto_approve
+                project,
+                path,
+                additional,
+                auto_approve,
+                extract,
+                estimate_rule,
+                join_rule,
+                export,
+                output,
             ) if external_file else import_dataset(
                 project, directory, extension, parse, additional
             )
@@ -452,18 +489,75 @@ def import_dataset(project, directory, extension, parse, additional):
         click.echo("Success!")
 
 
-def import_metafile(project, path, additional, auto_approve):
+def import_metafile(
+    project,
+    path,
+    additional,
+    auto_approve,
+    extract,
+    estimate_rule,
+    join_rule,
+    export,
+    output,
+):
     pjt = Project(project)
-
-    if path is None:
+    if (path == ()) and (join_rule is None):
         path = click.prompt(
             "Where is your meta-data file? (select a path for an external meta-data file)",
             type=str,
         )
     try:
-        pjt.add_metafile(
-            file_path=path, attributes=additional, auto=auto_approve, verbose=True
-        )
+        if extract:
+            for pth in path:
+                result = pjt.extract_metafile(
+                    file_path=pth, attributes=additional, verbose=2
+                )
+                if export is not None:
+                    if export.lower() == "csv":
+                        for i, res in enumerate(result, 1):
+                            result_keys = list(res[0].keys())
+                            for r in res:
+                                extra_keys = list(set(r.keys()) - set(result_keys))
+                                result_keys += extra_keys
+
+                            output_csv = ",".join(result_keys)
+                            for r in res:
+                                result_values = [str(r[k]) for k in result_keys]
+                                output_csv += "\n" + ",".join(result_values)
+
+                            output_path = os.path.join(
+                                ".",
+                                f"{os.path.basename(pth.split('.')[0])}_Table{i}.csv",
+                            )
+                            if output is not None:
+                                output_path = output
+                                os.makedirs(os.path.dirname(output), exist_ok=True)
+
+                            file_count = 1
+                            while True:
+                                if os.path.exists(output_path):
+                                    basename, ext = os.path.splitext(output_path)
+                                    output_path = f"{basename} ({file_count}){ext}"
+                                    file_count += 1
+                                else:
+                                    break
+                            with open(output_path, "w", encoding="utf-8") as f:
+                                f.write(output_csv)
+                    else:
+                        click.echo(
+                            f"Sorry, export file type: {export} was not supprted yet..."
+                        )
+        elif estimate_rule:
+            for pth in path:
+                pjt.estimate_join_rule(file_path=pth, verbose=2)
+        else:
+            pjt.add_metafile(
+                file_path=path,
+                attributes=additional,
+                auto=auto_approve,
+                join_rule_path=join_rule,
+                verbose=1,
+            )
     except Exception as e:
         print(e)
     else:
@@ -543,10 +637,22 @@ def search_files(
                     output_path = output
                     os.makedirs(os.path.dirname(output) or ".", exist_ok=True)
 
+                file_count = 1
+                while True:
+                    if os.path.exists(output_path):
+                        basename, ext = os.path.splitext(output_path)
+                        output_path = f"{basename} ({file_count}){ext}"
+                        file_count += 1
+                    else:
+                        break
                 with open(output_path, "w", encoding="utf-8") as f:
                     f.write(output_json)
             elif export.lower() == "csv":
-                result_keys = list(result[0].keys())
+                result_keys = []
+                for r in result:
+                    extra_keys = list(set(r.keys()) - set(result_keys))
+                    result_keys += extra_keys
+
                 output_csv = ",".join(result_keys)
                 for r in result:
                     result_values = [str(r[k]) for k in result_keys]
@@ -557,7 +663,15 @@ def search_files(
                     output_path = output
                     os.makedirs(os.path.dirname(output) or ".", exist_ok=True)
 
-                with open(output_path, "w") as f:
+                file_count = 1
+                while True:
+                    if os.path.exists(output_path):
+                        basename, ext = os.path.splitext(output_path)
+                        output_path = f"{basename} ({file_count}){ext}"
+                        file_count += 1
+                    else:
+                        break
+                with open(output_path, "w", encoding="utf-8") as f:
                     f.write(output_csv)
             else:
                 click.echo(f"Sorry, export file type: {export} was not supprted yet...")
diff --git a/base/files.py b/base/files.py
index 02d785d..524f789 100644
--- a/base/files.py
+++ b/base/files.py
@@ -278,7 +278,6 @@ def natural_keys(obj):
                         if key in data and eval(f"'{data[key]}' {operator} '{value}'"):
                             queried_result.append(data)
                 elif operator == "!=":
-                    # Obtains both data that is not the specified value and data that does not have the key attribute.
                     for data in result:
                         if key in data and not eval(
                             f"'{data[key]}' {operator} '{value}'"
@@ -298,9 +297,17 @@ def natural_keys(obj):
                             s = sorted([data[key], value], key=natural_keys)
                             if s[1] == value:
                                 queried_result.append(data)
-                elif operator in ["is", "is not"]:
+                elif operator == ["is", "is not"]:
+                    # in python, "is" and "is not" operators allowed to compare with `None`
+                    # so, if other values set as 'value', raise ValueError
+                    if value != "None":
+                        raise ValueError(
+                            "Only 'None' is allowed with `is` or `is not` operators."
+                        )
                     for data in result:
-                        if key in data and eval(f"{data[key]} {operator} {value}"):
+                        if (operator == "is" and key not in data) or (
+                            operator == "is not" and key in data
+                        ):
                             queried_result.append(data)
                 elif operator in ["in", "not in"]:
                     for data in result:
diff --git a/base/project.py b/base/project.py
index e28f2e9..65530b5 100644
--- a/base/project.py
+++ b/base/project.py
@@ -4,12 +4,14 @@
 # Please contact engineer@adansons.co.jp
 import os
 import json
+import ruamel.yaml
 import glob
 import math
 import base64
 import requests
-from typing import Optional, List, Tuple, Union
+from typing import Optional, List, Union
 import time
+from colorama import Fore, init
 
 from base.files import Files
 from base.parser import Parser
@@ -24,6 +26,8 @@
     BASE_API_ENDPOINT,
 )
 
+# colorama settings
+init(autoreset=True)
 
 HEADER = {"Content-Type": "application/json"}
 CONFIG_FILE = os.path.join(os.path.expanduser("~"), ".base", "projects")
@@ -426,16 +430,14 @@ def add_datafiles(
         file_num = len(data_list)
         return file_num
 
-    def add_metafile(
+    def extract_metafile(
         self,
         file_path: str,
         attributes: dict = {},
-        join_rule: dict = {},
-        auto: bool = False,
-        verbose: bool = True,
-    ) -> None:
+        verbose: int = 2,
+    ):
         """
-        Import meta data from external file.
+        Extract meta data from external file.
 
         Parameters
         ----------
@@ -443,18 +445,14 @@ def add_metafile(
             the external file path
         attributes : dict (default {})
             the extra meta data (attributes) combined with whole datafiles
-        join_rule : dict (default {})
-            the rule for table joining
-            {
-                "New table key 1": "Exist table key 1", <- if you have same key on new and exist tables
-                "New table key 2": "ADD:" + "Exist table key 2", <- if you have mew value on exist key
-                "New table key 3": None <- if you have new key
-            }
-        auto : bool (default False)
-            if True, skip to get confirmation
-        verbose : bool (default True)
-            if True, show detail of each actions result
-            if you turn off auto mode, you will always get detail for confirmation
+        verbose : int (default 2)
+            if verbose==2, show detail of each action result
+            if verbose==1, show summary of each action result
+
+        Returns
+        -------
+        tables: list
+            list of data extracted from external-file
 
         Raises
         ------
@@ -486,38 +484,190 @@ def add_metafile(
         s3_presigned_url = res.json()["URL"]
         res = requests.get(s3_presigned_url)
         tables = res.json()["Items"]
-        if verbose:
-            print(f"{len(tables)} tables found!")
-            print("now estimating the rule for table joining...\n")
 
-        # get update_rule for each table
-        if not join_rule:
-            update_rules = []
-            for table in tables:
-                url = f"{BASE_API_ENDPOINT}/project/{self.project_uid}?user={self.user_id}"
-                payload = {"Items": table}
-                res = requests.put(url, json.dumps(payload), headers=HEADER)
-                if res.status_code != 200:
-                    raise Exception("Failed to estimate the joining rule")
-
-                update_rule = res.json()["UpdateRule"]
-                update_rules.append(json.dumps(update_rule, ensure_ascii=False))
+        if verbose != 0:
+            print(f"{len(tables)} tables found! ({file_path})\n")
+        if verbose == 2:
+            for i, table in enumerate(tables, 1):
+                print(f"===== New Table{i} =====\n{summarize_parsed_table(table)}\n")
+        return tables
+
+    def estimate_join_rule(
+        self,
+        tables: Optional[list] = None,
+        file_path: Optional[str] = None,
+        verbose: int = 2,
+    ):
+        """
+        Estimate join rule from external file and existing table.
+
+        Parameters
+        ----------
+        tables : list
+            list of data extracted from external-file
+        file_path : str
+            the external file path
+        verbose : bool (default True)
+            if verbose==2, show detail of each action result
+            if verbose==1, show summary of each action result
+
+        Raises
+        ------
+        ValueError
+            raises if specified external file is not csv or excel file
+        Exception
+            raises if something went wrong on uploading request to server
+        """
+        if verbose in [0, 1, 2]:
+            print("now estimating the rule for table joining...")
+
+        if not (tables or file_path):
+            raise ValueError("You have to specify 'tables' or 'file_path'.")
+
+        if tables is None:
+            tables = []
+            _, ext = os.path.splitext(file_path)
+            if ext.lower() not in [".csv", ".xlsx"]:
+                raise ValueError(
+                    f"{ext} file is not supported. Currently only suports csv or xlsx file."
+                )
+            try:
+                with open(file_path, "r", encoding="utf-8") as f:
+                    csv_data = f.read()
+                key = csv_data.split("\n")[0].split(",")
+                values = csv_data.split("\n")[1:]
+                table = []
+                for value in values:
+                    table.append({key[i]: v for i, v in enumerate(value.split(","))})
+                tables.append(table)
+            except:
+                if verbose != 0:
+                    print(
+                        "Specified file looks like messy. Base will extract tables from it."
+                    )
+                extracted_tables = self.extract_metafile(file_path=file_path, verbose=1)
+                for extracted_table in extracted_tables:
+                    tables.append(extracted_table)
+
+        join_rules = []
+        for table in tables:
+            url = f"{BASE_API_ENDPOINT}/project/{self.project_uid}?user={self.user_id}"
+            payload = {"Items": table}
+            res = requests.put(url, json.dumps(payload), headers=HEADER)
+            if res.status_code != 200:
+                raise Exception("Failed to estimate the joining rule")
+
+            join_rule = res.json()["UpdateRule"]
+            join_rules.append(json.dumps(join_rule, ensure_ascii=False))
+
+        if verbose != 0:
+            print(f"{len(join_rules)} table joining rule was estimated! ({file_path})")
+        if verbose == 2:
+            for i, join_rule in enumerate(join_rules):
+                print(f"\nRule no.{i+1}")
+                for new_key, exist_key in json.loads(join_rule).items():
+                    if exist_key:
+                        print(
+                            f"\tkey '{new_key}'\t->\tconnected to '{exist_key}' key on exist table"
+                        )
+                    else:
+                        print(f"\tkey '{new_key}'\t->\tnewly added")
+                print(f"\nTable {i+1} sample record:\n\t{tables[i-1][0]}\n")
+
+        return join_rules
+
+    def add_metafile(
+        self,
+        file_path: Optional[tuple] = None,
+        attributes: dict = {},
+        join_rule: dict = {},
+        auto: bool = False,
+        join_rule_path: str = None,
+        verbose: int = 1,
+    ) -> None:
+        """
+        Import meta data from external file.
 
+        Parameters
+        ----------
+        file_path : str
+            the external file path
+        attributes : dict (default {})
+            the extra meta data (attributes) combined with whole datafiles
+        join_rule : dict (default {})
+            the rule for table joining
+            {
+                "New table key 1": "Exist table key 1", <- if you have same key on new and exist tables
+                "New table key 2": "ADD:" + "Exist table key 2", <- if you have new value on exist key
+                "New table key 3": None <- if you have new key
+            }
+        auto : bool (default False)
+            if True, skip to get confirmation
+        verbose : bool (default True)
+            if True, show detail of each action result
+            if you turn off auto mode, you will always get detail for confirmation
+
+        Raises
+        ------
+        ValueError
+            raises if specified external file is not csv or excel file or invalid YML file specified as join_rule_path
+        Exception
+            raises if something went wrong on uploading request to server
+        """
+        if join_rule_path:
+            try:
+                with open(join_rule_path, "r", encoding="utf-8") as yf:
+                    join_rules = ruamel.yaml.safe_load(yf)["Body"]
+                file_path = [rule["FilePath"] for rule in list(join_rules.values())]
+                file_path = sorted(set(file_path), key=file_path.index)
+            except:
+                raise ValueError("Invalid YAML file. Unable to read FilePath.")
+
+        tables = []
+        tables_from_path = []
+        for path in file_path:
+            # extract table from meta file
+            tables_ = self.extract_metafile(
+                file_path=path, attributes=attributes, verbose=verbose
+            )
+            tables_from_path += [path] * len(tables_)
+            tables += tables_
+
+        # get update_rule for each table
+        if not (join_rule or join_rule_path):
+            join_rules = self.estimate_join_rule(tables=tables, verbose=verbose)
             table_rule_pair = {}
-            for table, update_rule in zip(tables, update_rules):
-                if update_rule in table_rule_pair:
-                    table_rule_pair[update_rule].append(table)
+            for table, join_rule in zip(tables, join_rules):
+                if join_rule in table_rule_pair:
+                    table_rule_pair[join_rule].append(table)
                 else:
-                    table_rule_pair[update_rule] = [table]
-            if verbose:
-                print(f"{len(table_rule_pair)} table joining rule was estimated!")
-        else:
+                    table_rule_pair[join_rule] = [table]
+        elif join_rule:
             if len(tables) != 1:
                 raise ValueError(
                     "You can use join_rule option when you have only 1 table on external file."
                 )
             table_rule_pair = {json.dumps(join_rule, ensure_ascii=False): tables}
-        if verbose:
+        elif join_rule_path:
+            try:
+                with open(join_rule_path, "r", encoding="utf-8") as yf:
+                    join_rules = ruamel.yaml.safe_load(yf)["Body"]
+
+                join_rules = [
+                    json.dumps(rule["JoinRules"], ensure_ascii=False)
+                    for rule in list(join_rules.values())
+                ]
+                table_rule_pair = {}
+                for table, join_rule in zip(tables, join_rules):
+                    if join_rule in table_rule_pair:
+                        table_rule_pair[join_rule].append(table)
+                    else:
+                        table_rule_pair[join_rule] = [table]
+            except:
+                raise ValueError("Invalid YAML file.Unable to read JoinRules.")
+
+        if verbose == 1:
+            print(f"{len(table_rule_pair)} table joining rule was estimated!\n")
             print("Below table joining rule will be applied...\n\n")
             for i, update_rule in enumerate(list(table_rule_pair.keys())):
                 print(f"Rule no.{i+1}\n")
@@ -537,12 +687,16 @@ def add_metafile(
                 "\nDo you want to perform table join?\n\tBase will join tables with that rule described above.\n"
             )
             print("\t'y' will be accepted to approve.\n")
-            approved = True if input("\tEnter a value: ") == "y" else False
+            if not join_rule_path:
+                print(
+                    "\tIf you need to modify it, please enter 'm'\n\t\tDefinition YML file with estimated table join rules will be downloaded, then you can modify it and apply the new join rule."
+                )
+            approved = input("\tEnter a value: ")
         else:
-            approved = True
+            approved = "y"
 
         # update records
-        if approved:
+        if approved == "y":
             for update_rule, tables in table_rule_pair.items():
                 update_rule = json.loads(update_rule)
                 update_rule_for_add = {}
@@ -567,6 +721,91 @@ def add_metafile(
                     )
                     if res.status_code != 200:
                         raise Exception("Failed to join the tables")
+        elif approved == "m" and (not join_rule_path):
+            join_rules_info = {
+                "RequestedTime": time.time(),
+                "ProjectName": self.project_name,
+                "Body": {},
+            }
+            for i, rule in enumerate(join_rules, 1):
+                join_rules_info["Body"][f"Table{i}"] = {
+                    "FilePath": os.path.abspath(tables_from_path[i - 1]),
+                    "JoinRules": json.loads(rule),
+                }
+
+            yaml_str = ruamel.yaml.round_trip_dump(
+                join_rules_info, default_flow_style=False
+            )
+            yaml_str += """\n# [Description]
+                        \n# By modifying the Body/Table/JoinRules section, you can define a new join rule.
+                        \n# Fundamentally, this section consists of Key-Value Pairs. 
+                        \n# Key is the key name from the new table. Value is the key name from the existing table.\n
+                        \n# "New table key 1": "Exist table key 1", <- if you have same key on new and exist tables
+                        \n# "New table key 2": "ADD:" + "Exist table key 2", <- if you have new value on exist key
+                        \n# "New table key 3":   , <- if you have new key, no need to specify anything\n
+                        \n# [Example]
+                        \n#  JoinRules:
+                        \n#   first_name: name
+                        \n#   age: ADD:Age
+                        \n#   height:\n
+                        \n# The Key-Value above defines 3 join-rules. 
+                        \n# 1. "first_name: name" means to join the new key named "first_name" with the existing key named "name".
+                        \n#    If you have same key on the new and the existing tables, write like this.
+                        \n# 2. "age: ADD:Age" means to add new values of the new key named 'age' on the existing key named 'Age'.
+                        \n#    If you have new value on the existing key, write like this.
+                        \n# 3. "height: " means to add the key named "height" as a new key.
+                        \n#    If the new key is not in the existing table, write like this."""
+
+            yaml = ruamel.yaml.YAML()
+            yaml.default_flow_style = True
+            yaml_str = yaml.load(yaml_str)
+            file_name = f"joinrule_definition_{self.project_name}.yml"
+            file_count = 1
+            while True:
+                if os.path.exists(file_name):
+                    file_name = (
+                        f"joinrule_definition_{self.project_name} ({file_count}).yml"
+                    )
+                    file_count += 1
+                else:
+                    break
+            with open(file_name, "w", encoding="utf-8") as yf:
+                yaml.dump(yaml_str, yf)
+
+            print(
+                Fore.BLUE
+                + f"\nDownloaded a YAML file '{file_name}' in current directory.\n"
+                f"Key information for the new table and the existing table is as follows.\n\n"
+            )
+            for i, table in enumerate(tables, 1):
+                print(f"===== New Table{i} =====\n{summarize_parsed_table(table)}\n")
+
+            print(f"===== Existing Table =====")
+            summary_for_print = summarize_keys_information(self.get_metadata_summary())
+            max_len_list = [
+                summary_for_print["MaxCharCount"][column]
+                for column in summary_for_print["Keys"][0]
+            ]
+            for row in summary_for_print["Keys"]:
+                print(
+                    "  ".join(
+                        [
+                            content + " " * (length - len(content))
+                            for content, length in zip(row, max_len_list)
+                        ]
+                    )
+                )
+
+            attr_str = ""
+            if attributes:
+                for attr in list(attributes.items()):
+                    attr_str += " --additional " + ":".join(attr)
+
+            print(
+                Fore.BLUE + f"\nYou can apply the new join-rule according to 2 steps.\n"
+                f"1. Modify the file '{file_name}'. Open the file to see a detailed description.\n"
+                f"2. Execute the following command.\n   base import {self.project_name} --external-file{attr_str} --join-rule {file_name}\n"
+            )
         else:
             raise Exception("Aborted!")
 
@@ -577,7 +816,7 @@ def get_metadata_summary(self) -> List[dict]:
         Returns
         -------
         key_list : list
-            list of each keys information
+            list of each key information
             [
                 {
                     "KeyHash": String,
@@ -756,7 +995,7 @@ def get_members(self) -> List[dict]:
         Returns
         -------
         member_list : list
-            list of each members information
+            list of each member information
             [
                 {
                     "UserID": String,
@@ -1019,6 +1258,93 @@ def summarize_keys_information(metadata_summary: List[dict]) -> dict:
         },
         "Keys": summary_list,
     }
+
+    return summary_for_print
+
+
+def summarize_parsed_table(table: List[dict]) -> str:
+    """
+    Summarize information of extracted table from external file for printing.
+
+    Parameters
+    ----------
+    tables : list
+        list of data extracted from external-file
+
+    Returns
+    -------
+
+    summary_for_print : str
+        summarized table information
+    """
+
+    def select_vtype(vtype_list: list) -> str:
+        if "str" in vtype_list:
+            return "str"
+        elif "float" in vtype_list:
+            return "float"
+        elif "int" in vtype_list:
+            return "int"
+        elif "bool" in vtype_list:
+            return "bool"
+        else:
+            return "None"
+
+    dic = {}
+    for data in table:
+        for k, v in data.items():
+            if dic.get(k):
+                dic[k].append(v)
+            else:
+                dic[k] = [v]
+
+    table_summary = {}
+    for k in dic.keys():
+        unique_value = sorted(set(dic[k]))
+        key_summary = {
+            "UpperValue": unique_value[-1],
+            "LowerValue": unique_value[0],
+            "ValueType": select_vtype(
+                list(set(vt.__class__.__name__ for vt in unique_value))
+            ),
+            "RecordedCount": len(dic[k]),
+        }
+        table_summary[k] = key_summary
+
+    char_count = {
+        "KEY NAME": [8],  # length of "KEY NAME"
+        "VALUE RANGE": [11],  # length of "VALUE RANGE"
+        "VALUE TYPE": [10],  # length of "VALUE TYPE"
+        "RECORDED COUNT": [14],  # length of "RECORDED COUNT"
+    }
+    summary_list = [("KEY NAME", "VALUE RANGE", "VALUE TYPE", "RECORDED COUNT")]
+    for key_name, key_summary in table_summary.items():
+        value_range_summary = (
+            f'{key_summary["LowerValue"]} ~ {key_summary["UpperValue"]}'
+        )
+        value_type_summary = f"{key_summary['ValueType']}('{key_name}')"
+        summary = (
+            f"'{key_name}'",
+            value_range_summary,
+            value_type_summary,
+            str(key_summary["RecordedCount"]),
+        )
+        summary_list.append(summary)
+        char_count["KEY NAME"].append(len(f"'{key_name}'"))
+        char_count["VALUE RANGE"].append(len(value_range_summary))
+        char_count["VALUE TYPE"].append(len(value_type_summary))
+        char_count["RECORDED COUNT"].append(len(str(key_summary["RecordedCount"])))
+
+    max_len_list = [max(char_count[column]) for column in summary_list[0]]
+    summary_for_print = "\n".join(
+        "  ".join(
+            [
+                content + " " * (length - len(content))
+                for content, length in zip(row, max_len_list)
+            ]
+        )
+        for row in summary_list
+    )
     return summary_for_print
 
 
diff --git a/docs/CLI.md b/docs/CLI.md
index cb136e1..2ee6ad0 100644
--- a/docs/CLI.md
+++ b/docs/CLI.md
@@ -84,10 +84,14 @@ And then, Base will take below actions.
     
     >>> sample parsing rule: {{}}/{{name}}/{{timestamp}}/{{sensor}}-{{condition}}{{iteration}}.csv
     ```
-    
+The following options are used only when importing external files.
 - `-m`, `--external-file` - parse the content of external files which specified with `-p` option.
 - `-p <external-filepath>`, `--path <external-filepath>` - specify an `external-filepath` to import external files. Base will parse content of that file, extract table data on it, and parse the tables.
 - `-a <additional-key-value>`, `--additional <additional-key-value>` - specify additional meta data you want to add whole the file you import. the value must be include colon (”:”) between `key name` and `value string`. for instance, if you want to import and join an external file for only “test” data type files, you should specify like `-x dataType:test`.
+- `--extract` - with this option, only extract the content of external file, dose not link and update with existing tables. you can specify output path with `-e` and `-o` options to get extract results.
+  - `--export <export-file-type>` - if you want to convert extract results into CSV, you can specify CSV as export-file-type.
+  - `--output <output-filepath>`- specify output-filepath to save dataset file. default is “./{external-filepath}_Table{number}.csv”
+- `--estimate-rule` - with this option, only estimate the joining rule from existing tables and external files which specified with `-p` option, dose not link and update with existing tables.
 
 **Example: Import png files on project “mnist”**
 
@@ -103,6 +107,7 @@ $ base import mnist --directory ~/dataset/mnist --extension png --parse "{dataTy
 ```
 Check datafiles...
 found 70000 files with png extension.
+70000/70000 files uploaded.   
 Success!
 ```
 </details>
@@ -141,12 +146,96 @@ Do you want to perform table join?
         Base will join tables with that rule described above.
 
         'y' will be accepted to approve.
-
+        If you need to modify it, please enter 'm'
+                Definition YML file with estimated table join rules will be downloaded, then you can modify it and apply the new join rule.
         Enter a value: y
 Success!
 ```
+If you enter 'm', definition YAML file with estimated table join rules will be downloaded.  
+You can modify this file and execute the commands displayed in the terminal to apply the new join rule.  
+
+```
+Do you want to perform table join?
+        Base will join tables with that rule described above.
+
+        'y' will be accepted to approve.
+
+        If you need to modify it, please enter 'm'
+                Definition YML file with estimated table join rules will be downloaded, then you can modify it and apply the new join rule.
+        Enter a value: m
+
+Downloaded a YAML file 'joinrule_definition_mnist.yml' in current directory.
+Key information for the new table and the existing table is as follows.
+
+
+===== New Table1 =====
+KEY NAME         VALUE RANGE  VALUE TYPE            RECORDED COUNT
+'index'          8 ~ 9850     int('index')          74            
+'originalLabel'  0 ~ 9        int('originalLabel')  74            
+'correction'     -1 ~ 8or9    str('correction')     74            
+'dataType'       test ~ test  str('dataType')       74            
+
+===== Existing Table =====
+KEY NAME    VALUE RANGE   VALUE TYPE       RECORDED COUNT
+'id'        0 ~ 59999     str('id')        70000         
+'label'     0 ~ 9         str('label')     70000         
+'dataType'  test ~ train  str('dataType')  70000          
+
+You can apply the new join-rule according to 2 steps.
+1. Modify the file 'joinrule_definition_mnist.yml'. Open the file to see a detailed description.
+2. Execute the following command.
+   base import mnist --external-file --additional dataType:test --join-rule joinrule_definition_mnist.yml
+
+Success!
+```
+joinrule_definition_mnist.yml
+```yaml
+RequestedTime: 1654257223.4988642
+ProjectName: mnist
+Body:
+  Table1:
+    FilePath: /Users/user/Downloads/wrongImagesInMNISTTestset.csv
+    JoinRules:
+      index: id
+      originalLabel: label
+      correction:
+      dataType: dataType
+```
+New join rules can be defined by modifying the `Body/Table/JoinRules` section.  
+Fundamentally, this section consists of Key-Value Pairs. Key is the key name from the new table extracted from the external file. Value is the key name from the existing table.
+
+How to define join rules.  
+if you have same key on the new table and the existing table, write like this.
+```yaml
+ 'New table key': 'Existing table key'
+```
+
+if you have new value on the existing key, write like this.
+```yaml
+ 'New table key': 'ADD:Existing table key'
+```
+
+if you have new key, no need to specify anything.
+```yaml
+ 'New table key': 
+```
+
+For example:
+```yaml
+ JoinRules:
+  first_name: name
+  age: ADD:Age
+  height:
+```
+1. "first_name: name" means to join the new key named "first_name" with the existing key named "name".
+2. "age: ADD:Age" means to add new values of the new key named 'age' on the existing key named 'Age'.
+3. "height: " means to add the key named "height" as a new key.
+
+
+
 </details>
 
+
 → [Back to top](#command-reference)
 
 ## invite
@@ -653,7 +742,7 @@ And also you can export as JSON or CSV with `-e` and `-o` options.
     ```
     [query grammar]
     {KeyName} {Operator} {Values}
-    - ¥add 1 spaces each section
+    - add 1 spaces each section
     - don't use space any other
     >>> sample query condition: CategoryName == airplane
     
@@ -664,7 +753,10 @@ And also you can export as JSON or CSV with `-e` and `-o` options.
     - <= : less than
     - > : greater
     - < : less
+    - is : missing value (only 'None' is allowed as Values, ex. query='correction is None')
+    - is not : any value (only 'None' is allowed as Values, ex. query='correction is not None')
     - in : inner list of Values
+    - not in : outer list of values
     ```
     
     > Note:  you have to follow query grammar.
diff --git a/docs/SDK.md b/docs/SDK.md
index f3e28fe..aeb5c4a 100644
--- a/docs/SDK.md
+++ b/docs/SDK.md
@@ -773,6 +773,8 @@ These are the available methods:
 - [add_datafiles()](#adddatafiles)
 - [add_member()](#addmember)
 - [add_metafile()](#addmetafile)
+- [extract_metafile](#extractmetafile)
+- [estimate_join_rule](#estimatejoinrule)
 - [files()](#files)
 - [get_members()](#getmembers)
 - [get_metadata_summary()](#getmetadatasummary)
@@ -899,7 +901,29 @@ project.add_member(member="string", permission_level="string")
 Import meta data from external file.
 
 ```python
-project.add_metafile(file_path="string", attributes={"string":"string"})
+project.add_metafile(file_path=["string"], attributes={"string":"string"})
+```
+
+**Parameters**
+
+- file_path (list) - requeired
+    - list of the external file path
+- attributes (string) - default {}
+    - the extra meta data (attributes) combined with whole datafiles
+
+**Raises**
+
+- ValueError
+    - raises if specified external file is not csv or excel file
+- Exception
+    - raises if something went wrong on uploading request to server
+
+### **extract_metafile()**
+
+Only Extract meta data from external file.
+
+```python
+project.extract_metafile(file_path="string", attributes={"string":"string"})
 ```
 
 **Parameters**
@@ -909,6 +933,64 @@ project.add_metafile(file_path="string", attributes={"string":"string"})
 - attributes (string) - default {}
     - the extra meta data (attributes) combined with whole datafiles
 
+**Returns**
+- tables (list)
+    - list of table data extracted from external file
+
+```JavaScript
+[
+    [
+        {
+            "MetaKey1": ...,
+            "MetaKey2": ...,
+            ...
+        },
+        ...
+    ],
+    ...
+]
+```
+
+**Raises**
+
+- ValueError
+    - raises if specified external file is not csv or excel file
+- Exception
+    - raises if something went wrong on uploading request to server
+
+### **estimate_join_rule()**
+
+Only estimate the join rule from external file and existing table.
+
+```python
+project.extract_metafile(file_path="string", tables=list)
+```
+
+**Parameters**
+
+Either file_path or tables are required. If both are specified, tables take precedence.
+- file_path (string)
+    - the external file path
+- tables (list)
+    - output of base.Project().extract_metafile() method  
+
+
+
+**Returns**
+- join_rule (list)
+    - list of the join rule estimated from external file and existing table.
+
+```JavaScript
+[
+        {
+            "new key1":"exist key1" ...,
+            ...
+        },
+        ...
+]
+```
+
+
 **Raises**
 
 - ValueError
@@ -918,7 +1000,7 @@ project.add_metafile(file_path="string", attributes={"string":"string"})
 
 ### **files()**
 
-Return the `[Files class](#files-class)`.
+Return the [`Files class`](#files-class).
 You can filter files easily and simply by specified criteria.
 
 ```python
@@ -982,7 +1064,7 @@ files = project.files(conditions="string", query=["string"], sort_key="string")
 
 **Returns**
 
-- `[Files class](#files-class)`
+- [`Files class`](#files-class)
 
 ### **get_members()**
 
diff --git a/poetry.lock b/poetry.lock
index 0b9d941..5c24225 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1068,6 +1068,14 @@ category = "dev"
 optional = false
 python-versions = ">=3.7"
 
+[[package]]
+name = "pyyaml"
+version = "6.0"
+description = "YAML parser and emitter for Python"
+category = "main"
+optional = false
+python-versions = ">=3.6"
+
 [[package]]
 name = "pyzmq"
 version = "22.3.0"
@@ -1134,6 +1142,29 @@ urllib3 = ">=1.21.1,<1.27"
 socks = ["PySocks (>=1.5.6,!=1.5.7)", "win-inet-pton"]
 use_chardet_on_py3 = ["chardet (>=3.0.2,<5)"]
 
+[[package]]
+name = "ruamel.yaml"
+version = "0.17.21"
+description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order"
+category = "main"
+optional = false
+python-versions = ">=3"
+
+[package.dependencies]
+"ruamel.yaml.clib" = {version = ">=0.2.6", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.11\""}
+
+[package.extras]
+docs = ["ryd"]
+jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"]
+
+[[package]]
+name = "ruamel.yaml.clib"
+version = "0.2.6"
+description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml"
+category = "main"
+optional = false
+python-versions = ">=3.5"
+
 [[package]]
 name = "s3transfer"
 version = "0.5.2"
@@ -1374,7 +1405,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.7"
-content-hash = "b570fdca8a261485af34c3e5623ac6f79e2fbd195bbba47f3402c5ffa57773bd"
+content-hash = "bb1eb4b9ec00470a9ed0a488b681ac3204f0912bcdf8bf8da3d8a31370f9bb45"
 
 [metadata.files]
 anyio = [
@@ -1930,6 +1961,41 @@ pywinpty = [
     {file = "pywinpty-2.0.5-cp39-none-win_amd64.whl", hash = "sha256:e59a508ae78374febada3e53b5bbc90b5ad07ae68cbfd72a2e965f9793ae04f3"},
     {file = "pywinpty-2.0.5.tar.gz", hash = "sha256:e125d3f1804d8804952b13e33604ad2ca8b9b2cac92b27b521c005d1604794f8"},
 ]
+pyyaml = [
+    {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"},
+    {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"},
+    {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"},
+    {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"},
+    {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"},
+    {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"},
+    {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"},
+    {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"},
+    {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"},
+    {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"},
+    {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"},
+    {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"},
+    {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"},
+    {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"},
+    {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"},
+    {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"},
+    {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"},
+    {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"},
+    {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"},
+    {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"},
+    {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"},
+    {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"},
+    {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"},
+    {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"},
+    {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"},
+    {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"},
+    {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"},
+    {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"},
+    {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"},
+    {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"},
+    {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"},
+    {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"},
+    {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"},
+]
 pyzmq = [
     {file = "pyzmq-22.3.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:6b217b8f9dfb6628f74b94bdaf9f7408708cb02167d644edca33f38746ca12dd"},
     {file = "pyzmq-22.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2841997a0d85b998cbafecb4183caf51fd19c4357075dfd33eb7efea57e4c149"},
@@ -1991,6 +2057,37 @@ requests = [
     {file = "requests-2.27.1-py2.py3-none-any.whl", hash = "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"},
     {file = "requests-2.27.1.tar.gz", hash = "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61"},
 ]
+"ruamel.yaml" = [
+    {file = "ruamel.yaml-0.17.21-py3-none-any.whl", hash = "sha256:742b35d3d665023981bd6d16b3d24248ce5df75fdb4e2924e93a05c1f8b61ca7"},
+    {file = "ruamel.yaml-0.17.21.tar.gz", hash = "sha256:8b7ce697a2f212752a35c1ac414471dc16c424c9573be4926b56ff3f5d23b7af"},
+]
+"ruamel.yaml.clib" = [
+    {file = "ruamel.yaml.clib-0.2.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6e7be2c5bcb297f5b82fee9c665eb2eb7001d1050deaba8471842979293a80b0"},
+    {file = "ruamel.yaml.clib-0.2.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:221eca6f35076c6ae472a531afa1c223b9c29377e62936f61bc8e6e8bdc5f9e7"},
+    {file = "ruamel.yaml.clib-0.2.6-cp310-cp310-win32.whl", hash = "sha256:1070ba9dd7f9370d0513d649420c3b362ac2d687fe78c6e888f5b12bf8bc7bee"},
+    {file = "ruamel.yaml.clib-0.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:77df077d32921ad46f34816a9a16e6356d8100374579bc35e15bab5d4e9377de"},
+    {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-macosx_10_6_intel.whl", hash = "sha256:cfdb9389d888c5b74af297e51ce357b800dd844898af9d4a547ffc143fa56751"},
+    {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:7b2927e92feb51d830f531de4ccb11b320255ee95e791022555971c466af4527"},
+    {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-win32.whl", hash = "sha256:ada3f400d9923a190ea8b59c8f60680c4ef8a4b0dfae134d2f2ff68429adfab5"},
+    {file = "ruamel.yaml.clib-0.2.6-cp35-cp35m-win_amd64.whl", hash = "sha256:de9c6b8a1ba52919ae919f3ae96abb72b994dd0350226e28f3686cb4f142165c"},
+    {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d67f273097c368265a7b81e152e07fb90ed395df6e552b9fa858c6d2c9f42502"},
+    {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:72a2b8b2ff0a627496aad76f37a652bcef400fd861721744201ef1b45199ab78"},
+    {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-win32.whl", hash = "sha256:9efef4aab5353387b07f6b22ace0867032b900d8e91674b5d8ea9150db5cae94"},
+    {file = "ruamel.yaml.clib-0.2.6-cp36-cp36m-win_amd64.whl", hash = "sha256:846fc8336443106fe23f9b6d6b8c14a53d38cef9a375149d61f99d78782ea468"},
+    {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0847201b767447fc33b9c235780d3aa90357d20dd6108b92be544427bea197dd"},
+    {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:78988ed190206672da0f5d50c61afef8f67daa718d614377dcd5e3ed85ab4a99"},
+    {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-win32.whl", hash = "sha256:a49e0161897901d1ac9c4a79984b8410f450565bbad64dbfcbf76152743a0cdb"},
+    {file = "ruamel.yaml.clib-0.2.6-cp37-cp37m-win_amd64.whl", hash = "sha256:bf75d28fa071645c529b5474a550a44686821decebdd00e21127ef1fd566eabe"},
+    {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a32f8d81ea0c6173ab1b3da956869114cae53ba1e9f72374032e33ba3118c233"},
+    {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7f7ecb53ae6848f959db6ae93bdff1740e651809780822270eab111500842a84"},
+    {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-win32.whl", hash = "sha256:89221ec6d6026f8ae859c09b9718799fea22c0e8da8b766b0b2c9a9ba2db326b"},
+    {file = "ruamel.yaml.clib-0.2.6-cp38-cp38-win_amd64.whl", hash = "sha256:31ea73e564a7b5fbbe8188ab8b334393e06d997914a4e184975348f204790277"},
+    {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dc6a613d6c74eef5a14a214d433d06291526145431c3b964f5e16529b1842bed"},
+    {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:1866cf2c284a03b9524a5cc00daca56d80057c5ce3cdc86a52020f4c720856f0"},
+    {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-win32.whl", hash = "sha256:3fb9575a5acd13031c57a62cc7823e5d2ff8bc3835ba4d94b921b4e6ee664104"},
+    {file = "ruamel.yaml.clib-0.2.6-cp39-cp39-win_amd64.whl", hash = "sha256:825d5fccef6da42f3c8eccd4281af399f21c02b32d98e113dbc631ea6a6ecbc7"},
+    {file = "ruamel.yaml.clib-0.2.6.tar.gz", hash = "sha256:4ff604ce439abb20794f05613c374759ce10e3595d1867764dd1ae675b85acbd"},
+]
 s3transfer = [
     {file = "s3transfer-0.5.2-py3-none-any.whl", hash = "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971"},
     {file = "s3transfer-0.5.2.tar.gz", hash = "sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed"},
diff --git a/pyproject.toml b/pyproject.toml
index d9faa6d..2783a79 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -16,6 +16,9 @@ click = "==8.0.4"
 requests = ">=1.0.0"
 numpy = ">=1.15"
 scikit-learn = ">=0.10"
+PyYAML = "^6.0"
+"ruamel.yaml" = "^0.17.21"
+colorama = "^0.4.4"
 
 [tool.poetry.dev-dependencies]
 black = "^21.12b0"


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 9ea4850..f7d572d 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -99,6 +99,38 @@ def test_import_dataset():
     assert "Success!" in result.output
 
 
+def test_import_metafile_extract():
+    runner = CliRunner()
+    result = runner.invoke(
+        import_data,
+        [
+            PROJECT_NAME,
+            "-m",
+            "-p",
+            os.path.join(os.path.dirname(__file__), "data", "sample.xlsx"),
+            "--extract",
+        ],
+    )
+    assert result.exit_code == 0
+    assert "Success!" in result.output
+
+
+def test_import_metafile_estimate_rule():
+    runner = CliRunner()
+    result = runner.invoke(
+        import_data,
+        [
+            PROJECT_NAME,
+            "-m",
+            "-p",
+            os.path.join(os.path.dirname(__file__), "data", "sample.xlsx"),
+            "--estimate-rule",
+        ],
+    )
+    assert result.exit_code == 0
+    assert "Success!" in result.output
+
+
 def test_import_metafile():
     runner = CliRunner()
     result = runner.invoke(
@@ -107,7 +139,7 @@ def test_import_metafile():
             PROJECT_NAME,
             "-m",
             "-p",
-            os.path.dirname(__file__) + "/data/sample.xlsx",
+            os.path.join(os.path.dirname(__file__), "data", "sample.xlsx"),
             "--auto-approve",
         ],
     )
@@ -115,6 +147,38 @@ def test_import_metafile():
     assert "Success!" in result.output
 
 
+def test_import_metafile_modify():
+    runner = CliRunner()
+    result = runner.invoke(
+        import_data,
+        [
+            PROJECT_NAME,
+            "-m",
+            "-p",
+            os.path.join(os.path.dirname(__file__), "data", "sample.xlsx"),
+        ],
+        input="m",
+    )
+    assert result.exit_code == 0
+    assert "Success!" in result.output
+
+
+def test_import_metafile_modify_join_rule_file():
+    runner = CliRunner()
+    result = runner.invoke(
+        import_data,
+        [
+            PROJECT_NAME,
+            "-m",
+            "--join-rule",
+            "joinrule_definition_adansons_test_project.yml",
+        ],
+        input="y",
+    )
+    assert result.exit_code == 0
+    assert "Success!" in result.output
+
+
 def test_import_metafile_exkeyvalue():
     runner = CliRunner()
     result = runner.invoke(
@@ -123,7 +187,7 @@ def test_import_metafile_exkeyvalue():
             PROJECT_NAME,
             "-m",
             "-p",
-            os.path.dirname(__file__) + "/data/sample.xlsx",
+            os.path.join(os.path.dirname(__file__), "data", "sample.xlsx"),
             "-a",
             "key1:value1",
             "--auto-approve",
@@ -141,7 +205,7 @@ def test_import_metafile_exkeyvalue_multiple():
             PROJECT_NAME,
             "-m",
             "-p",
-            os.path.dirname(__file__) + "/data/sample.xlsx",
+            os.path.join(os.path.dirname(__file__), "data", "sample.xlsx"),
             "-a",
             "key1:value1",
             "-a",
@@ -161,7 +225,7 @@ def test_import_metafile_exkeyvalue_invalid():
             PROJECT_NAME,
             "-m",
             "-p",
-            os.path.dirname(__file__) + "/data/sample.xlsx",
+            os.path.join(os.path.dirname(__file__), "data", "sample.xlsx"),
             "-a",
             "key1-value1",
             "--auto-approve",
@@ -179,7 +243,7 @@ def test_import_metafile_csv():
             PROJECT_NAME,
             "-m",
             "-p",
-            os.path.dirname(__file__) + "/data/sample.csv",
+            os.path.join(os.path.dirname(__file__), "data", "sample.csv"),
             "--auto-approve",
         ],
     )
@@ -195,7 +259,7 @@ def test_import_metafile_csv_exkeyvalue():
             PROJECT_NAME,
             "-m",
             "-p",
-            os.path.dirname(__file__) + "/data/sample.csv",
+            os.path.join(os.path.dirname(__file__), "data", "sample.csv"),
             "-a",
             "key1:value1",
             "--auto-approve",
@@ -213,7 +277,7 @@ def test_import_metafile_csv_exkeyvalue_multiple():
             PROJECT_NAME,
             "-m",
             "-p",
-            os.path.dirname(__file__) + "/data/sample.csv",
+            os.path.join(os.path.dirname(__file__), "data", "sample.csv"),
             "-a",
             "key1:value1",
             "-a",
@@ -249,7 +313,7 @@ def test_import_metafile_csv_exkeyvalue_invalid():
             PROJECT_NAME,
             "-m",
             "-p",
-            os.path.dirname(__file__) + "/data/sample.csv",
+            os.path.join(os.path.dirname(__file__), "data", "sample.csv"),
             "-a",
             "key1-value1",
         ],
@@ -344,7 +408,11 @@ def test_delete_project():
     test_list_project()
     test_show_project_detail()
     test_import_dataset()
+    test_import_metafile_extract()
+    test_import_metafile_estimate_rule()
     test_import_metafile()
+    test_import_metafile_modify()
+    test_import_metafile_modify_join_rule_file()
     test_import_metafile_exkeyvalue()
     test_import_metafile_exkeyvalue_multiple()
     test_import_metafile_csv_exkeyvalue()
diff --git a/tests/test_project.py b/tests/test_project.py
index 73f3221..c8c1161 100644
--- a/tests/test_project.py
+++ b/tests/test_project.py
@@ -173,14 +173,26 @@ def test_add_datafiles():
 
 def test_add_datafile():
     project = Project(PROJECT_NAME)
-    file_path = TESTS_DIR + "/data/sample.jpeg"
+    file_path = os.path.join(TESTS_DIR, "data", "sample.jpeg")
     attributes = {"title": "sample"}
     file_num = project.add_datafile(file_path, attributes)
 
 
+def test_extract_metafile():
+    project = Project(PROJECT_NAME)
+    file_path = os.path.join(TESTS_DIR, "data", "sample.xlsx")
+    project.extract_metafile(file_path)
+
+
+def test_estimate_join_rule():
+    project = Project(PROJECT_NAME)
+    file_path = os.path.join(TESTS_DIR, "data", "sample.xlsx")
+    project.estimate_join_rule(file_path=file_path)
+
+
 def test_add_metafile():
     project = Project(PROJECT_NAME)
-    file_path = TESTS_DIR + "/data/sample.xlsx"
+    file_path = [os.path.join(TESTS_DIR, "data", "sample.xlsx")]
     project.add_metafile(file_path, auto=True)
 
 
@@ -236,6 +248,8 @@ def test_summarize_keys_information():
     test_get_projects()
     test_add_datafiles()
     test_add_datafile()
+    test_extract_metafile()
+    test_estimate_join_rule()
     test_add_metafile()
     test_get_metadata_summary()
     test_link_datafiles()