Repository: RedHatInsights/insights-core

Issue:
The validation of ContextProvider for core collection failed
For the `insights-archive` collected by the new Core Collection identified by the `"insights_archive.txt"`, the path of the real collected data is placed in the './data' but not the './' which works for the old archives.


It always reports the following error when `insights-run` against a new archive:

```

Traceback (most recent call last):
  File "/Users/liuxc/work/insights/insights-core/insights/core/plugins.py", line 90, in invoke
    return self.component(broker)
  File "/Users/liuxc/work/insights/insights-core/insights/core/spec_factory.py", line 569, in __call__
    return self.kind(ctx.locate_path(self.path), root=ctx.root, ds=self, ctx=ctx)
  File "/Users/liuxc/work/insights/insights-core/insights/core/spec_factory.py", line 175, in __init__
    self.validate()
  File "/Users/liuxc/work/insights/insights-core/insights/core/spec_factory.py", line 183, in validate
    raise ContentException("%s does not exist." % self.path)
insights.core.plugins.ContentException: /private/tmp/insights-ip-172-31-21-63.us-east-2.compute.internal-20210713080914/etc/redhat-release does not exist.
```

But in fact, the file is placed at 
```
insights-ip-172-31-21-63.us-east-2.compute.internal-20210713080914/data/etc/redhat-release
```


Discussion:
This issue can be simply reproduced by running any rule against an insights-archive collected with the new core collection with `insights-run -t`:

```
# insights-run -p insights/plugins ~/insights-ip-172-31-21-63.us-east-2.compute.internal-20210712064946.tar.gz -t | grep redhat-release -B10

Traceback (most recent call last):
  File "/home/liuxc/work/insights/insights-core/insights/core/plugins.py", line 90, in invoke
    return self.component(broker)
  File "/home/liuxc/work/insights/insights-core/insights/core/spec_factory.py", line 569, in __call__
    return self.kind(ctx.locate_path(self.path), root=ctx.root, ds=self, ctx=ctx)
  File "/home/liuxc/work/insights/insights-core/insights/core/spec_factory.py", line 175, in __init__
    self.validate()
  File "/home/liuxc/work/insights/insights-core/insights/core/spec_factory.py", line 183, in validate
    raise ContentException("%s does not exist." % self.path)
insights.core.plugins.ContentException: /tmp/insights-bcswfw8d/insights-ip-172-31-21-63.us-east-2.compute.internal-20210712064946/etc/redhat-release does not exist.
```

Resolving patch:
diff --git a/insights/core/dr.py b/insights/core/dr.py
--- a/insights/core/dr.py
+++ b/insights/core/dr.py
@@ -65,6 +65,7 @@ def add(a, b):
 from insights.contrib import importlib
 from insights.contrib.toposort import toposort_flatten
 from insights.core.blacklist import BLACKLISTED_SPECS
+from insights.core.context import SerializedArchiveContext
 from insights.core.exceptions import BlacklistedSpec, MissingRequirements, SkipComponent
 from insights.util import defaults, enum, KeyPassingDefaultDict
 
@@ -1087,6 +1088,14 @@ def run(components=None, broker=None):
     components = components or COMPONENTS[GROUPS.single]
     components = determine_components(components)
     broker = broker or Broker()
+    # If a SerializedArchiveContext then data found in the archive's
+    # ./meta_data directory are prepopulated in the broker as Specs so
+    # no need to collect them again
+    if broker.get(SerializedArchiveContext) is not None:
+        for comp in list(components):
+            if comp in broker:
+                for dep in components[comp]:
+                    components.pop(dep, None)
     return run_components(run_order(components), components, broker)
 
 
diff --git a/insights/core/spec_factory.py b/insights/core/spec_factory.py
--- a/insights/core/spec_factory.py
+++ b/insights/core/spec_factory.py
@@ -13,7 +13,7 @@
 from subprocess import call
 
 from insights.core import blacklist, dr
-from insights.core.context import ExecutionContext, FSRoots, HostContext
+from insights.core.context import FSRoots, ExecutionContext, HostContext, SerializedArchiveContext
 from insights.core.exceptions import BlacklistedSpec, ContentException, SkipComponent
 from insights.core.filters import _add_filter, get_filters
 from insights.core.plugins import component, datasource, is_datasource
@@ -128,14 +128,22 @@ def load(self):
 class FileProvider(ContentProvider):
     def __init__(self, relative_path, root="/", ds=None, ctx=None):
         super(FileProvider, self).__init__()
-        self.root = root
+        self.ds = ds
+        self.ctx = ctx
+        self._set_root(root)
         self.relative_path = relative_path.lstrip("/")
         self.file_name = os.path.basename(self.path)
 
-        self.ds = ds
-        self.ctx = ctx
         self.validate()
 
+    def _set_root(self, root):
+        if (isinstance(self.ctx, SerializedArchiveContext) and os.path.basename(root) != 'data'):
+            # For core-collection (SerializedArchiveContext) archive, data are
+            # in `/data` directory
+            self.root = os.path.join(root, 'data')
+        else:
+            self.root = root
+
     def validate(self):
         if not blacklist.allow_file("/" + self.relative_path):
             log.warning("WARNING: Skipping file %s", "/" + self.relative_path)
@@ -177,9 +185,10 @@ def _stream(self):
             raise ContentException(str(ex))
 
     def validate(self):
-        # Do NOT need to validate metadata files, they are newly generated
-        # by the core collection itself
-        pass
+        # Validate built-in metedata files only when insights-run
+        if not isinstance(self.ctx, HostContext):
+            super(MetadataProvider, self).validate()
+        # But DO NOT validate them when core-collecting
 
     def load(self):
         self.loaded = True
@@ -187,7 +196,8 @@ def load(self):
             return [l.rstrip("\n") for l in f]
 
     def write(self, dst):
-        # TODO: the metadata files can also be collected via core collection
+        # TODO: the built-ine metadata files can also be collected via
+        #       core-collection
         pass
 
 
@@ -1288,6 +1298,8 @@ def deserialize_datasource_provider(_type, data, root):
 
 @serializer(MetadataProvider)
 def serialize_metadata_provider(obj, root):
+    # Built-in metadata files are put in the root instead of '/data'
+    root = os.path.dirname(root) if os.path.basename(root) == 'data' else root
     dst = os.path.join(root, obj.relative_path.lstrip("/"))
     fs.ensure_path(os.path.dirname(dst))
     obj.write(dst)
@@ -1296,7 +1308,7 @@ def serialize_metadata_provider(obj, root):
 
 @deserializer(MetadataProvider)
 def deserialize_metadata_provider(_type, data, root):
-    # metadata files are put in the root directory instead of '/data'
+    # Built-in metadata files are put in the root instead of '/data'
     root = os.path.dirname(root) if os.path.basename(root) == 'data' else root
     return SerializedOutputProvider(data["relative_path"], root)
 


Test patch: