Repository: SoftwareUnderstanding/inspect4py (language: python)

Issue:
Potential bug when inspecting directories
In line 1252:https://github.com/SoftwareUnderstanding/inspect4py/blob/60622d1064e925bab57b01e7a0ab5511d4d2433a/inspect4py/cli.py#L1250-L1253 we are using the subdirectory's base name as the output subdirectory name. For example, when inspecting `Chowlk`, we have the json files of all python file under `Chowlk/` stored in `<output>/Chowlk/json_files/`, then we have the json files of all python file under `Chowlk/source/chowlk/` stored in `<output>/chowlk/json_files/`.

However, if we have two subdirectory of the same name this could be a problem, for example, if we have both `Chowlk/source/chowlk/` and `Chowlk/source/sub/chowlk/`, as they cannot be stored in the same output path `<output>/chowlk/json_files/` the latter one is going to be ignored in line 1166:https://github.com/SoftwareUnderstanding/inspect4py/blob/60622d1064e925bab57b01e7a0ab5511d4d2433a/inspect4py/cli.py#L1164-L1171

Also, when running inspect4py on MacOS, since the operating system path is not case-sensitive, when inspecting `Chowlk`, the output of `Chowlk/source/chowlk/` is already ignored, as python consider `<output>/chowlk/json_files/` already exists when it is actually `<output>/Chowlk/json_files/` being created before. Hence there is not json files under `Chowlk/source/chowlk/` in the output directory


Discussion / hints:

Fix patch:
diff --git a/.gitignore b/.gitignore
index cc2e161..150e5c1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,3 +16,4 @@ output_dir/
 defoe/*
 env_*
 inspect4py.egg-info/*
+test/test_files/hellogitworld
diff --git a/README.md b/README.md
index f3a1091..7b30dbc 100644
--- a/README.md
+++ b/README.md
@@ -235,6 +235,9 @@ Options:
   -ast, -—abstract_syntax_tree    generates abstract syntax tree in json format.
   -sc, --source_code              generates source code of each ast node.
   -ld, --license_detection        detects the license of the target repository.
+  -rm, --readme                   extract all readme files in the target repository.
+  -md, --metadata                 extract metadata of the target repository using
+                                  Github API.
   --help                          Show this message and exit.
 ```
 
diff --git a/docs/usage.md b/docs/usage.md
index c341872..3b26bc3 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -47,5 +47,8 @@ Options:
   -ast, -—abstract_syntax_tree    generates abstract syntax tree in json format.
   -sc, --source_code              generates source code of each ast node.
   -ld, --license_detection        detects the license of the target repository.
+  -rm, --readme                   extract all readme files in the target repository.
+  -md, --metadata                 extract metadata of the target repository using
+                                  Github API.
   --help                          Show this message and exit.
 ```
\ No newline at end of file
diff --git a/inspect4py/cli.py b/inspect4py/cli.py
index 66ad2ca..e77bd94 100644
--- a/inspect4py/cli.py
+++ b/inspect4py/cli.py
@@ -1200,8 +1200,13 @@ def create_output_dirs(output_dir, control_flow):
               help="generates the source code of each ast node.")
 @click.option('-ld', '--license_detection', type=bool, is_flag=True,
               help="detects the license of the target repository.")
+@click.option('-rm', '--readme', type=bool, is_flag=True,
+              help="extract all readme files in the target repository.")
+@click.option('-md', '--metadata', type=bool, is_flag=True, 
+              help="extract metadata of the target repository using Github API. (requires repository to have the .git folder)")
 def main(input_path, fig, output_dir, ignore_dir_pattern, ignore_file_pattern, requirements, html_output, call_list,
-         control_flow, directory_tree, software_invocation, abstract_syntax_tree, source_code, license_detection):
+         control_flow, directory_tree, software_invocation, abstract_syntax_tree, source_code, license_detection, readme,
+         metadata):
     if (not os.path.isfile(input_path)) and (not os.path.isdir(input_path)):
         print('The file or directory specified does not exist')
         sys.exit()
@@ -1249,7 +1254,8 @@ def main(input_path, fig, output_dir, ignore_dir_pattern, ignore_file_pattern, r
                 if ".py" in f and not f.endswith(".pyc"):
                     try:
                         path = os.path.join(subdir, f)
-                        out_dir = output_dir + "/" + os.path.basename(subdir) # TODO: This could create path like /parent//child
+                        relative_path = Path(subdir).relative_to(Path(input_path).parent)
+                        out_dir = str(Path(output_dir) / relative_path)
                         cf_dir, json_dir = create_output_dirs(out_dir, control_flow)
                         code_info = CodeInspection(path, cf_dir, json_dir, fig, control_flow, abstract_syntax_tree, source_code)
                         if code_info.fileJson:
@@ -1305,6 +1311,10 @@ def main(input_path, fig, output_dir, ignore_dir_pattern, ignore_file_pattern, r
                 dir_info["detected_license"] = [{k: f"{v:.1%}"} for k, v in rank_list]
             except:
                 pass
+        if readme:
+            dir_info["readme_files"] = extract_readme(input_path)
+        if metadata:
+            dir_info["metadata"] = get_github_metadata(input_path)
         json_file = output_dir + "/directory_info.json"
         pruned_json = prune_json(dir_info)
         with open(json_file, 'w') as outfile:
diff --git a/inspect4py/utils.py b/inspect4py/utils.py
index f8e2856..fff5098 100644
--- a/inspect4py/utils.py
+++ b/inspect4py/utils.py
@@ -1,6 +1,8 @@
 import ast
 import os
 import re
+import git
+import requests
 import subprocess
 from pathlib import Path
 
@@ -693,3 +695,49 @@ def detect_license(input_path, licenses_path, threshold=0.9):
         return sorted(rank_list, key=lambda t: t[1], reverse=True)
 
     return "License not recognised"
+
+
+def extract_readme(input_path: str) -> dict:
+    """
+    Function to extract content of all readme file under the input directory.
+    :param input_path: Path of the repository to be analyzed.
+    """
+    readme_files = {}
+    for file in Path(input_path).rglob("README.*"):
+        try:
+            with open(file, 'r') as f:
+                readme_files[str(file)] = f.read()
+        except Exception as e:
+            print(f"Error when opening {file}: {e}")
+
+    return readme_files
+
+
+def get_github_metadata(input_path: str) -> dict:
+    """
+    Function to extract metadata from the remote repository using Github api.
+    It requires connectivity to the Github API and the local target repository 
+    to have .git folder and a remote repository on Github.
+
+    :param input_path: Path of the repository to be analyzed.
+    """
+    github_metadata = {}
+    try:
+        repo = git.Repo(input_path)
+        remote_url = repo.remotes.origin.url
+
+        # Extract owner and repo name from remote url
+        api_param = re.search(r".+github.com[:/](?P<param>.+).git", remote_url).group("param")
+
+        # Call Github API to get the metadata
+        api_url = f"https://api.github.com/repos/{api_param}"
+        response = requests.get(api_url)
+        github_metadata = response.json()
+    except git.InvalidGitRepositoryError as e:
+        print(f"{input_path}.git not found or not valid: {e}")
+    except git.NoSuchPathError as e:
+        print(f"{input_path} does not exist: {e}")
+    except requests.exceptions.RequestException as e:
+        print(f"Error when accessing {api_url}: {e}")
+
+    return github_metadata
diff --git a/requirements.txt b/requirements.txt
index 3456bda..f7b3acd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,3 +8,4 @@ setuptools==54.2.0
 json2html
 configparser
 bigcode_astgen
+GitPython


Test patch:
diff --git a/test/test_files/test_readme/README.md b/test/test_files/test_readme/README.md
new file mode 100644
index 0000000..3f06fad
--- /dev/null
+++ b/test/test_files/test_readme/README.md
@@ -0,0 +1,1 @@
+README.md in root dir
diff --git a/test/test_files/test_readme/subdir/README.txt b/test/test_files/test_readme/subdir/README.txt
new file mode 100644
index 0000000..606ea82
--- /dev/null
+++ b/test/test_files/test_readme/subdir/README.txt
@@ -0,0 +1,1 @@
+README.txt in subdir
diff --git a/test/test_files/test_readme/subdir/subsubdir/README.rst b/test/test_files/test_readme/subdir/subsubdir/README.rst
new file mode 100644
index 0000000..f3ca390
--- /dev/null
+++ b/test/test_files/test_readme/subdir/subsubdir/README.rst
@@ -0,0 +1,1 @@
+README.rst in subsubdir
diff --git a/test/test_inspect4py.py b/test/test_inspect4py.py
index 5e3c7cf..1f9b1d5 100644
--- a/test/test_inspect4py.py
+++ b/test/test_inspect4py.py
@@ -1,5 +1,6 @@
 import unittest
 import shutil
+import requests
 from inspect4py.cli import *
 from inspect4py import cli, utils
 
@@ -268,8 +269,11 @@ def test_service(self):
         abstract_syntax_tree = False
         source_code = False
         license_detection = False
+        readme = False
+        metadata = False
         dir_info = invoke_inspector(input_path, fig, output_dir, ignore_dir_pattern, ignore_file_pattern, requirements,
-                                    call_list, control_flow, directory_tree, software_invocation, abstract_syntax_tree, source_code, license_detection)
+                                    call_list, control_flow, directory_tree, software_invocation, abstract_syntax_tree, 
+                                    source_code, license_detection, readme, metadata)
         current_type = dir_info['software_type']
         shutil.rmtree(output_dir)
         assert current_type[0]["type"] == "service"
@@ -288,8 +292,11 @@ def test_package(self):
         abstract_syntax_tree = False
         source_code = False
         license_detection = False
+        readme = False
+        metadata = False
         dir_info = invoke_inspector(input_path, fig, output_dir, ignore_dir_pattern, ignore_file_pattern, requirements,
-                                    call_list, control_flow, directory_tree, software_invocation, abstract_syntax_tree, source_code, license_detection)
+                                    call_list, control_flow, directory_tree, software_invocation, abstract_syntax_tree, 
+                                    source_code, license_detection, readme, metadata)        
         current_type = dir_info['software_type']
         shutil.rmtree(output_dir)
         assert current_type[0]["type"] == "package"
@@ -308,8 +315,11 @@ def test_library(self):
         abstract_syntax_tree = False
         source_code = False
         license_detection = False
+        readme = False
+        metadata = False
         dir_info = invoke_inspector(input_path, fig, output_dir, ignore_dir_pattern, ignore_file_pattern, requirements,
-                                    call_list, control_flow, directory_tree, software_invocation, abstract_syntax_tree, source_code, license_detection)
+                                    call_list, control_flow, directory_tree, software_invocation, abstract_syntax_tree, 
+                                    source_code, license_detection, readme, metadata)
         current_type = dir_info['software_type']
         shutil.rmtree(output_dir)
         assert current_type[0]["type"] == "library"
@@ -329,8 +339,11 @@ def test_multiple_mains(self):
         abstract_syntax_tree = False
         source_code = False
         license_detection = False
+        readme = False
+        metadata = False
         dir_info = invoke_inspector(input_path, fig, output_dir, ignore_dir_pattern, ignore_file_pattern, requirements,
-                                    call_list, control_flow, directory_tree, software_invocation, abstract_syntax_tree, source_code, license_detection)
+                                    call_list, control_flow, directory_tree, software_invocation, abstract_syntax_tree,
+                                    source_code, license_detection, readme, metadata)
         imports = dir_info['software_invocation']
         shutil.rmtree(output_dir)
         assert len(imports[0]["imports"]) == 2
@@ -351,8 +364,11 @@ def test_script(self):
         abstract_syntax_tree = False
         source_code = False
         license_detection = False
+        readme = False
+        metadata = False
         dir_info = invoke_inspector(input_path, fig, output_dir, ignore_dir_pattern, ignore_file_pattern, requirements,
-                                    call_list, control_flow, directory_tree, software_invocation, abstract_syntax_tree, source_code, license_detection)
+                                    call_list, control_flow, directory_tree, software_invocation, abstract_syntax_tree,
+                                    source_code, license_detection, readme, metadata)
         current_type = dir_info['software_type']
         shutil.rmtree(output_dir)
         assert current_type[0]["type"] == "script"
@@ -531,19 +547,93 @@ def test_license_detection(self):
         abstract_syntax_tree = False
         source_code = False
         license_detection = True
+        readme = False
+        metadata = False
 
         expected_liceses = ['Apache-2.0', 'LGPL-3.0', 'MIT']
         first_rank_licenses = []
         for input_path in input_paths:
             dir_info = invoke_inspector(input_path, fig, output_dir, ignore_dir_pattern, ignore_file_pattern, requirements,
-                                    call_list, control_flow, directory_tree, software_invocation, abstract_syntax_tree, source_code, license_detection)
+                                    call_list, control_flow, directory_tree, software_invocation, abstract_syntax_tree, 
+                                    source_code, license_detection, readme, metadata)
             first_rank_licenses.append(next(iter(dir_info["detected_license"][0])))
             shutil.rmtree(output_dir)
         
         assert first_rank_licenses == expected_liceses
 
+
+    def test_readme(self):
+        input_path = "./test_files/test_readme"
+        output_dir = "./output_dir"
+        fig = False
+        ignore_dir_pattern = [".", "__pycache__"]
+        ignore_file_pattern = [".", "__pycache__"]
+        requirements = False
+        call_list = False
+        control_flow = False
+        directory_tree = False
+        software_invocation = False
+        abstract_syntax_tree = False
+        source_code = False
+        license_detection = False
+        readme = True
+        metadata = False
+
+        dir_info = invoke_inspector(input_path, fig, output_dir, ignore_dir_pattern, ignore_file_pattern, requirements,
+                                    call_list, control_flow, directory_tree, software_invocation, abstract_syntax_tree, 
+                                    source_code, license_detection, readme, metadata)
+
+        root_dir = Path(input_path)
+        expected_readme_files = {
+            f"{root_dir}/README.md": "README.md in root dir\n",
+            f"{root_dir}/subdir/README.txt": "README.txt in subdir\n",
+            f"{root_dir}/subdir/subsubdir/README.rst": "README.rst in subsubdir\n"
+        }
+        actual_readme_files = dir_info["readme_files"]
+        print(actual_readme_files)
+        assert expected_readme_files == actual_readme_files        
+
+
+    def test_metadata(self):
+        """ 
+        Need to execute under test/test_files/: 
+        `git clone https://github.com/githubtraining/hellogitworld.git`
+        to pass this test, as getting metadata requires the local repository
+        to have a .git folder.
+        """
+        input_path = "./test_files/hellogitworld"
+        output_dir = "./output_dir"
+        fig = False
+        ignore_dir_pattern = [".", "__pycache__"]
+        ignore_file_pattern = [".", "__pycache__"]
+        requirements = False
+        call_list = False
+        control_flow = False
+        directory_tree = False
+        software_invocation = False
+        abstract_syntax_tree = False
+        source_code = False
+        license_detection = False
+        readme = False
+        metadata = True
+
+        dir_info = invoke_inspector(input_path, fig, output_dir, ignore_dir_pattern, ignore_file_pattern, requirements,
+                                    call_list, control_flow, directory_tree, software_invocation, abstract_syntax_tree, 
+                                    source_code, license_detection, readme, metadata)
+        try:
+            response = requests.get("https://api.github.com/repos/githubtraining/hellogitworld")
+            expected_metadata = response.json()
+        except requests.RequestException as e:
+            print(f"Error sending requests to Github API: {e}")
+            raise e
+
+        actual_metadata = dir_info["metadata"]
+        assert expected_metadata == actual_metadata        
+
+
 def invoke_inspector(input_path, fig, output_dir, ignore_dir_pattern, ignore_file_pattern, requirements,
-                     call_list, control_flow, directory_tree, software_invocation, abstract_syntax_tree, source_code, license_detection):
+                     call_list, control_flow, directory_tree, software_invocation, abstract_syntax_tree,
+                     source_code, license_detection, readme, metadata):
     dir_info = {}
     # retrieve readme text at the root level (if any)
     readme = ""
@@ -563,7 +653,8 @@ def invoke_inspector(input_path, fig, output_dir, ignore_dir_pattern, ignore_fil
             if ".py" in f and not f.endswith(".pyc"):
                 try:
                     path = os.path.join(subdir, f)
-                    out_dir = output_dir + "/" + os.path.basename(subdir)
+                    relative_path = Path(subdir).relative_to(Path(input_path).parent)
+                    out_dir = str(Path(output_dir) / relative_path)
                     cf_dir, json_dir = create_output_dirs(out_dir, control_flow)
                     code_info = CodeInspection(path, cf_dir, json_dir, fig, control_flow, abstract_syntax_tree, source_code)
                     if out_dir not in dir_info:
@@ -612,6 +703,10 @@ def invoke_inspector(input_path, fig, output_dir, ignore_dir_pattern, ignore_fil
             licenses_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../inspect4py/licenses")
             rank_list = detect_license(input_path, licenses_path)
             dir_info["detected_license"] = [{k: f"{v:.1%}"} for k, v in rank_list]
+    if readme:
+        dir_info["readme_files"] = extract_readme(input_path)
+    if metadata:
+        dir_info["metadata"] = get_github_metadata(input_path)
     return dir_info