Repository: tianocore/edk2-pytool-library

Issue:
stuart_ci_build does not support execution from paths other than the workspace root
stuart_ci_build is working when I execute it from the workspace root, but fails when I run it from a subdirectory.

For example, the following works from the Mu_Plus repo workspace root:
```mu_plus>stuart_ci_build -c .pytool\CISettings.py -p FirmwarePolicyPkg```

But when I change to a subdirectory:
```mu_plus\FirmwarePolicyPkg>stuart_ci_build -c ..\.pytool\CISettings.py -p FirmwarePolicyPkg```

it fails with
```
ERROR - --->Test Failed: Compiler Plugin DEBUG returned 1
PROGRESS - --Running FirmwarePolicyPkg: Compiler Plugin RELEASE --
CRITICAL - EXCEPTION: [Errno 2] No such file or directory: 'FirmwarePolicyPkg/FirmwarePolicyPkg.dsc'
CRITICAL - Traceback (most recent call last):
   File "c:\git\venv_1911\lib\site-packages\edk2toolext\invocables\edk2_ci_build.py", line 226, in Go
    tc, plugin_output_stream)
   File "C:\git\FwPolicy\fw_policy\MU_BASECORE\.pytool\Plugin\CompilerPlugin\CompilerPlugin.py", line 80, in RunBuildPlugin
    dp.ParseFile(AP_Path)
   File "c:\git\venv_1911\lib\site-packages\edk2toollib\uefi\edk2\parsers\dsc_parser.py", line 259, in ParseFile
    f = open(os.path.join(filepath), "r")
 FileNotFoundError: [Errno 2] No such file or directory: 'FirmwarePolicyPkg/FirmwarePolicyPkg.dsc'

ERROR - --->Test Failed: Compiler Plugin RELEASE returned 1
```


Comments:

Patch:
diff --git a/edk2toollib/uefi/edk2/parsers/base_parser.py b/edk2toollib/uefi/edk2/parsers/base_parser.py
index d2179d1..2974082 100644
--- a/edk2toollib/uefi/edk2/parsers/base_parser.py
+++ b/edk2toollib/uefi/edk2/parsers/base_parser.py
@@ -7,6 +7,7 @@
 ##
 import os
 import logging
+from edk2toollib.uefi.edk2 import path_utilities
 
 
 class BaseParser(object):
@@ -24,8 +25,8 @@ def __init__(self, log="BaseParser"):
         self.ConditionalStack = []
         self.RootPath = ""
         self.PPs = []
-        self.TargetFile = None
-        self.TargetFilePath = None
+        self._Edk2PathUtil = None
+        self.TargetFilePath = None  # the abs path of the target file
         self.CurrentLine = -1
         self._MacroNotDefinedValue = "0"  # value to used for undefined macro
 
@@ -42,19 +43,27 @@ def SetBaseAbsPath(self, path):
         Returns:
 
         """
-        self.RootPath = path
+        self.RootPath = os.path.abspath(path)
+        self._ConfigEdk2PathUtil()
         return self
 
+    def _ConfigEdk2PathUtil(self):
+        ''' creates the path utility object based on the root path and package paths '''
+        self._Edk2PathUtil = path_utilities.Edk2Path(self.RootPath, self.PPs, error_on_invalid_pp=False)
+
     def SetPackagePaths(self, pps=[]):
         """
 
         Args:
           pps:  (Default value = [])
 
+        This must be called after SetBaseAbsPath
+
         Returns:
 
         """
         self.PPs = pps
+        self._ConfigEdk2PathUtil()
         return self
 
     def SetInputVars(self, inputdict):
@@ -71,37 +80,43 @@ def SetInputVars(self, inputdict):
 
     def FindPath(self, *p):
         """
-
+        Given a path, it will find it relative to the root, the current target file, or the packages path
         Args:
-          *p:
+          *p: any number of strings or path like objects
 
-        Returns:
+        Returns: a full absolute path if the file exists, None on failure
 
         """
-        # NOTE: Some of this logic should be replaced
-        #       with the path resolution from Edk2Module code.
+        # check if we're getting a None
+        if p is None or (len(p) == 1 and p[0] is None):
+            return None
+
+        Path = os.path.join(*p)
+        # check if it it is absolute
+        if os.path.isabs(Path) and os.path.exists(Path):
+            return Path
 
         # If the absolute path exists, return it.
         Path = os.path.join(self.RootPath, *p)
         if os.path.exists(Path):
-            return Path
+            return os.path.abspath(Path)
 
         # If that fails, check a path relative to the target file.
         if self.TargetFilePath is not None:
-            Path = os.path.join(self.TargetFilePath, *p)
+            Path = os.path.abspath(os.path.join(os.path.dirname(self.TargetFilePath), *p))
             if os.path.exists(Path):
-                return Path
+                return os.path.abspath(Path)
 
         # If that fails, check in every possible Pkg path.
-        for Pkg in self.PPs:
-            Path = os.path.join(self.RootPath, Pkg, *p)
-            if os.path.exists(Path):
+        if self._Edk2PathUtil is not None:
+            target_path = os.path.join(*p)
+            Path = self._Edk2PathUtil.GetAbsolutePathOnThisSytemFromEdk2RelativePath(target_path, False)
+            if Path is not None:
                 return Path
 
         # log invalid file path
-        Path = os.path.join(self.RootPath, *p)
-        self.Logger.error("Invalid file path %s" % Path)
-        return Path
+        self.Logger.error(f"Invalid file path: {p}")
+        return None
 
     def WriteLinesToFile(self, filepath):
         """
diff --git a/edk2toollib/uefi/edk2/parsers/dsc_parser.py b/edk2toollib/uefi/edk2/parsers/dsc_parser.py
index bc73ba4..d783100 100644
--- a/edk2toollib/uefi/edk2/parsers/dsc_parser.py
+++ b/edk2toollib/uefi/edk2/parsers/dsc_parser.py
@@ -47,9 +47,12 @@ def __ParseLine(self, Line, file_name=None, lineno=None):
         if(line_resolved.strip().lower().startswith("!include")):
             # include line.
             tokens = line_resolved.split()
-            self.Logger.debug("Opening Include File %s" % os.path.join(self.RootPath, tokens[1]))
-            sp = self.FindPath(tokens[1])
-            self._dsc_file_paths.add(sp)
+            include_file = tokens[1]
+            sp = self.FindPath(include_file)
+            if sp is None:
+                raise FileNotFoundError(include_file)
+            self.Logger.debug("Opening Include File %s" % sp)
+            self._PushTargetFile(sp)
             lf = open(sp, "r")
             loc = lf.readlines()
             lf.close()
@@ -182,8 +185,12 @@ def __ParseDefineLine(self, Line):
         if(line_resolved.strip().lower().startswith("!include")):
             # include line.
             tokens = line_resolved.split()
-            self.Logger.debug("Opening Include File %s" % os.path.join(self.RootPath, tokens[1]))
-            sp = self.FindPath(tokens[1])
+            include_file = tokens[1]
+            self.Logger.debug("Opening Include File %s" % include_file)
+            sp = self.FindPath(include_file)
+            if sp is None:
+                raise FileNotFoundError(include_file)
+            self._PushTargetFile(sp)
             lf = open(sp, "r")
             loc = lf.readlines()
             lf.close()
@@ -297,20 +304,25 @@ def ResetParserState(self):
 
     def ParseFile(self, filepath):
         self.Logger.debug("Parsing file: %s" % filepath)
-        self.TargetFile = os.path.abspath(filepath)
-        self.TargetFilePath = os.path.dirname(self.TargetFile)
-        sp = os.path.join(filepath)
-        self._dsc_file_paths.add(sp)
+        sp = self.FindPath(filepath)
+        if sp is None:
+            raise FileNotFoundError(filepath)
+        self._PushTargetFile(sp)
         f = open(sp, "r")
         # expand all the lines and include other files
         file_lines = f.readlines()
         self.__ProcessDefines(file_lines)
         # reset the parser state before processing more
         self.ResetParserState()
+        self._PushTargetFile(sp)
         self.__ProcessMore(file_lines, file_name=sp)
         f.close()
         self.Parsed = True
 
+    def _PushTargetFile(self, targetFile):
+        self.TargetFilePath = os.path.abspath(targetFile)
+        self._dsc_file_paths.add(self.TargetFilePath)
+
     def GetMods(self):
         return self.ThreeMods + self.SixMods
 
diff --git a/edk2toollib/uefi/edk2/path_utilities.py b/edk2toollib/uefi/edk2/path_utilities.py
index a4ebdce..ea76674 100644
--- a/edk2toollib/uefi/edk2/path_utilities.py
+++ b/edk2toollib/uefi/edk2/path_utilities.py
@@ -103,7 +103,7 @@ def GetEdk2RelativePathFromAbsolutePath(self, abspath):
         self.logger.error("AbsolutePath: %s" % abspath)
         return None
 
-    def GetAbsolutePathOnThisSytemFromEdk2RelativePath(self, relpath):
+    def GetAbsolutePathOnThisSytemFromEdk2RelativePath(self, relpath, log_errors=True):
         ''' Given a edk2 relative path return an absolute path to the file
         in this workspace.
 
@@ -124,8 +124,9 @@ def GetAbsolutePathOnThisSytemFromEdk2RelativePath(self, relpath):
             abspath = os.path.join(a, relpath)
             if(os.path.exists(abspath)):
                 return abspath
-        self.logger.error("Failed to convert Edk2Relative Path to an Absolute Path on this system.")
-        self.logger.error("Relative Path: %s" % relpath)
+        if log_errors:
+            self.logger.error("Failed to convert Edk2Relative Path to an Absolute Path on this system.")
+            self.logger.error("Relative Path: %s" % relpath)
 
         return None
 


Test patch:
diff --git a/edk2toollib/uefi/edk2/parsers/base_parser_test.py b/edk2toollib/uefi/edk2/parsers/base_parser_test.py
index 6f3f191..a7857ff 100644
--- a/edk2toollib/uefi/edk2/parsers/base_parser_test.py
+++ b/edk2toollib/uefi/edk2/parsers/base_parser_test.py
@@ -588,26 +588,34 @@ def test_find_path(self):
         parser.Lines = ["hello"]
         package_paths = ["Common/Test", "SM_MAGIC"]
         root_path = tempfile.mkdtemp()
-        target_filedir = os.path.join(root_path, "BuildPkg")
-        parser.TargetFilePath = target_filedir
-        parser.SetPackagePaths(package_paths)
-        parser.SetBaseAbsPath(root_path)
-        os.makedirs(target_filedir)
         index = 0
-        root_file = "root.txt"
-        target_file = "target.txt"
+        # create the packages path folders
         for package in package_paths:
             pack_path = os.path.join(root_path, package)
             os.makedirs(pack_path)
             parser.WriteLinesToFile(os.path.join(pack_path, f"package_{index}.txt"))
             index += 1
+        # setup the parser
+        parser.SetBaseAbsPath(root_path)
+        parser.SetPackagePaths(package_paths)
+
+        # create the root and target files
+        root_file = "root.txt"
+        target_file = "target.txt"
+
         root_filepath = os.path.join(root_path, root_file)
+        target_filedir = os.path.join(root_path, "BuildPkg")
         target_filepath = os.path.join(target_filedir, target_file)
+        # create root file
         parser.WriteLinesToFile(root_filepath)
+        # create target file
+        os.makedirs(target_filedir)
         parser.WriteLinesToFile(target_filepath)
-
+        parser.TargetFilePath = target_filepath
+        # check if we can find the root
         root_found = parser.FindPath(root_file)
         self.assertEqual(root_found, root_filepath)
+        # check we can find the target using the target path
         target_found = parser.FindPath(target_file)
         self.assertEqual(target_found, target_filepath)
 
@@ -620,8 +628,12 @@ def test_find_path(self):
         # invalid files
         invalid_filename = "YOU_WONT_FIND_ME.txt"
         invalid_file = os.path.join(root_path, invalid_filename)
-        invalid_result = parser.FindPath(invalid_filename)
-        self.assertEqual(invalid_file, invalid_result)
+        invalid_result = parser.FindPath(invalid_file)
+        invalid_result2 = parser.FindPath(invalid_filename)
+        self.assertEqual(None, invalid_result)
+        self.assertEqual(None, invalid_result2)
+        invalid_result3 = parser.FindPath(None)
+        self.assertEqual(None, invalid_result3)
 
     # make sure we can write out to a file
 
diff --git a/edk2toollib/uefi/edk2/parsers/dsc_parser_test.py b/edk2toollib/uefi/edk2/parsers/dsc_parser_test.py
index 5d2a124..5e6296c 100644
--- a/edk2toollib/uefi/edk2/parsers/dsc_parser_test.py
+++ b/edk2toollib/uefi/edk2/parsers/dsc_parser_test.py
@@ -30,7 +30,7 @@ def write_to_file(file_path, data):
 
     def test_dsc_include_single_file(self):
         ''' This tests whether includes work properly '''
-        workspace = tempfile.gettempdir()
+        workspace = tempfile.mkdtemp()
 
         file1_name = "file1.dsc"
         file2_name = "file2.dsc"
@@ -54,7 +54,7 @@ def test_dsc_include_single_file(self):
 
     def test_dsc_include_missing_file(self):
         ''' This tests whether includes work properly '''
-        workspace = tempfile.gettempdir()
+        workspace = tempfile.mkdtemp()
 
         file1_name = "file1.dsc"
         file1_path = os.path.join(workspace, file1_name)
@@ -70,7 +70,7 @@ def test_dsc_include_missing_file(self):
 
     def test_dsc_include_missing_file_no_fail_mode(self):
         ''' This tests whether includes work properly if no fail mode is on'''
-        workspace = tempfile.gettempdir()
+        workspace = tempfile.mkdtemp()
 
         file1_name = "file1.dsc"
         file1_path = os.path.join(workspace, file1_name)
@@ -83,3 +83,74 @@ def test_dsc_include_missing_file_no_fail_mode(self):
         parser.SetNoFailMode()
         parser.SetBaseAbsPath(workspace)
         parser.ParseFile(file1_path)
+
+    def test_dsc_parse_file_on_package_path(self):
+        ''' This tests whether includes work properly if no fail mode is on'''
+        workspace = tempfile.mkdtemp()
+        working_dir_name = "working"
+        working2_dir_name = "working2"
+
+        working_folder = os.path.join(workspace, working_dir_name)
+        working2_folder = os.path.join(working_folder, working2_dir_name)
+        os.makedirs(working_folder, exist_ok=True)
+        os.makedirs(working2_folder, exist_ok=True)
+
+        file1_name = "file1.dsc"
+        file1_path = os.path.join(working2_folder, file1_name)
+        file1_short_path = os.path.join(working2_dir_name, file1_name)
+        file1_data = "[Defines]\n INCLUDED=TRUE"
+
+        TestDscParserIncludes.write_to_file(file1_path, file1_data)
+        with self.assertRaises(FileNotFoundError):
+            parser = DscParser()
+            parser.SetBaseAbsPath(workspace)
+            parser.ParseFile(file1_short_path)
+
+        parser = DscParser()
+        parser.SetBaseAbsPath(workspace)
+        parser.SetPackagePaths([working_folder, ])
+        parser.ParseFile(file1_short_path)
+        self.assertEqual(parser.LocalVars["INCLUDED"], "TRUE")  # make sure we got the defines
+
+    def test_dsc_include_relative_path(self):
+        ''' This tests whether includes work properly with a relative path'''
+        workspace = tempfile.mkdtemp()
+        outside_folder = os.path.join(workspace, "outside")
+        inside_folder = os.path.join(outside_folder, "inside")
+        inside2_folder = os.path.join(outside_folder, "inside2")
+        random_folder = os.path.join(outside_folder, "random")
+        os.makedirs(inside_folder, exist_ok=True)
+        os.makedirs(inside2_folder, exist_ok=True)
+        os.makedirs(random_folder, exist_ok=True)
+        cwd = os.getcwd()
+        os.chdir(random_folder)
+        try:
+
+            file1_name = "file1.dsc"
+            file1_path = os.path.join(outside_folder, file1_name)
+
+            file2_name = "file2.dsc"
+            file2_path = os.path.join(inside_folder, file2_name)
+
+            file3_name = "file3.dsc"
+            file3_path = os.path.join(inside2_folder, file3_name)
+
+            file1_data = "!include " + os.path.relpath(file2_path, os.path.dirname(file1_path)).replace("\\", "/")
+            file2_data = "!include " + os.path.relpath(file3_path, os.path.dirname(file2_path)).replace("\\", "/")
+            file3_data = "[Defines]\n INCLUDED=TRUE"
+
+            print(f"{file1_path}: {file1_data}")
+            print(f"{file2_path}: {file2_data}")
+            print(f"{file3_path}: {file3_data}")
+
+            TestDscParserIncludes.write_to_file(file1_path, file1_data)
+            TestDscParserIncludes.write_to_file(file2_path, file2_data)
+            TestDscParserIncludes.write_to_file(file3_path, file3_data)
+
+            parser = DscParser()
+            parser.SetBaseAbsPath(workspace)
+            parser.ParseFile(file1_path)
+
+            self.assertEqual(parser.LocalVars["INCLUDED"], "TRUE")  # make sure we got the defines
+        finally:
+            os.chdir(cwd)