Repository: andrewDoing/hydrate

Issue:
Add path to fabrikate subcomponent
For components that have a source in a github repo, add a path property with the corresponding path instead of including the full link to the source.

Comments:

Patch:
diff --git a/README.md b/README.md
index df64bf4..084fde5 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 Hydrate crawls a kubernetes cluster and generates a high level description of your deployments.
 
 ## Setup
-Ensure you are using Python 3 or a newer version.
+Ensure you are using Python 3.6 or a newer version.
 Include a "kubeconfig" file for your cluster in the same directory as hydrate.py,
 or specify one with the -k argument.
 Finally, install the dependencies.
@@ -12,8 +12,15 @@ pip install -r requirements.txt
 
 ## Basic Usage
 ```bash
-python -m hydrate [-h] [-n NAME] [-o path] [-v] [-d] run
+python -m hydrate [-h] [-n NAME] [-k FILE] [-o PATH] [-v] [-d] run
 ```
+The component.yaml file that is created is based on the specification detailed in the [Fabrikate](https://github.com/Microsoft/fabrikate "Fabrikate") repo.
+
+[Fabrikate Component Definition](https://github.com/microsoft/fabrikate/blob/master/docs/component.md "Component Definition")
+
+[Fabrikate Config Definition](https://github.com/microsoft/fabrikate/blob/master/docs/config.md "Config Definition")
+
+
 
 ### Positional arguments:
 
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index bd33118..99166bc 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -10,8 +10,6 @@ pool:
   vmImage: 'ubuntu-latest'
 strategy:
   matrix:
-    Python35:
-      python.version: '3.5'
     Python36:
       python.version: '3.6'
     Python37:
diff --git a/hydrate/component.py b/hydrate/component.py
index 96c1736..3b929d4 100644
--- a/hydrate/component.py
+++ b/hydrate/component.py
@@ -6,9 +6,9 @@ from copy import deepcopy
 class Component():
     """Hold the information for fabrikate High-Level Deployment(HLD)."""
 
-    def __init__(self, name, generator="static",
-                 source="<source repository url>", method="git",
-                 path=None):
+    def __init__(self, name, generator=None, source="<source repository url>",
+                 method="git", path=None, version=None, branch=None,
+                 hooks=None, repositories=None, subcomponents=None):
         """Instantiate a Component object.
 
         Args:
@@ -22,11 +22,11 @@ class Component():
         self.source = source
         self.method = method
         self.path = path
-        self.version = None
-        self.branch = None
-        self.hooks = None
-        self.repositories = None
-        self.subcomponents = None
+        self.version = version
+        self.branch = branch
+        self.hooks = hooks
+        self.repositories = repositories
+        self.subcomponents = subcomponents
 
     def __eq__(self, other):
         """Override the default __eq__."""
diff --git a/hydrate/scrape.py b/hydrate/scrape.py
index eec595d..88623d7 100644
--- a/hydrate/scrape.py
+++ b/hydrate/scrape.py
@@ -5,28 +5,55 @@ import re
 from .component import Component
 
 # URL to the Fabrikate Component Definitions
-COMP_DEFS_URL = "https://api.github.com/repos/microsoft/fabrikate-definitions/contents/definitions"
+FAB_DEFS_URL = "https://github.com/microsoft/fabrikate-definitions"
+FAB_DEFS_API = "https://api.github.com/repos/microsoft/fabrikate-definitions/contents/definitions"
 
 
 def get_repo_components():
     """Return the Fabrikate Component List."""
-    json_obj = json_get(COMP_DEFS_URL)
+    json_obj = json_get(FAB_DEFS_API)
     if json_obj:
-        components = parse_json(json_obj)
+        json_data = parse_json(json_obj)
+        components = construct_components(json_data)
         components = remove_fabrikate_prefix(components)
         return components
-    raise Exception('JSON not retrieved. URL:{}'.format(COMP_DEFS_URL))
+    raise Exception('JSON not retrieved. URL:{}'.format(FAB_DEFS_API))
 
 
-def parse_json(json_list):
-    """Parse json to get each component."""
+def get_path(html_url):
+    """Get the component path from the html_url."""
+    return re.sub(r'.*master/', '', html_url)
+
+
+def construct_components(json_data):
+    """Construct Component objects using a list of data tuples."""
     components = []
-    for entry in json_list:
-        component = Component(entry["name"], source=entry["html_url"])
-        components.append(component)
+    for defintion in json_data:
+        components.append(
+            Component(name=defintion["name"],
+                      source=defintion["source"],
+                      path=defintion["path"]))
     return components
 
 
+def parse_json(json_list):
+    """Parse json to get information for each definition.
+
+    Returns:
+        dict
+
+    """
+    json_dicts = []
+    for entry in json_list:
+        json_data = {
+                     'name': entry["name"],
+                     'source': FAB_DEFS_URL,
+                     'path': get_path(entry["html_url"])
+                    }
+        json_dicts.append(json_data)
+    return json_dicts
+
+
 def remove_fabrikate_prefix(components):
     """Remove the fabrikate prefix from the Component names."""
     for component in components:


Test patch:
diff --git a/tests/test_scrape.py b/tests/test_scrape.py
index c945d5f..2f11ee8 100644
--- a/tests/test_scrape.py
+++ b/tests/test_scrape.py
@@ -2,8 +2,10 @@
 import pytest
 
 from hydrate.component import Component
+from hydrate.scrape import FAB_DEFS_URL, FAB_DEFS_API
 from hydrate.scrape import get_repo_components
 from hydrate.scrape import parse_json
+from hydrate.scrape import construct_components
 from hydrate.scrape import remove_fabrikate_prefix
 from hydrate.scrape import json_get
 
@@ -11,31 +13,64 @@ from hydrate.scrape import json_get
                          [(1), (None)])
 def test_get_repo_components(mocker, json_get_ret):
     """Test the get_repo_components function."""
+    mock_json_get = mocker.patch("hydrate.scrape.json_get",
+                                 return_value=json_get_ret)
     mock_parse_json = mocker.patch("hydrate.scrape.parse_json",
                                    return_value=json_get_ret)
+    mock_construct_components = mocker.patch(
+        "hydrate.scrape.construct_components",
+        return_value=json_get_ret)
     mock_rm_fab_prefix = mocker.patch("hydrate.scrape.remove_fabrikate_prefix")
-    mock_json_get = mocker.patch("hydrate.scrape.json_get")
-    get_repo_components()
+
+    if json_get_ret:
+        get_repo_components()
+    else:
+        with pytest.raises(Exception, match=r".* URL:%s" % (FAB_DEFS_API)):
+            get_repo_components()
+
     mock_json_get.assert_called_once()
 
     if mock_json_get.return_value:
         mock_parse_json.assert_called_once()
+        mock_construct_components.assert_called_once()
         mock_rm_fab_prefix.assert_called_once()
     else:
         mock_parse_json.assert_not_called()
+        mock_construct_components.assert_not_called()
         mock_rm_fab_prefix.assert_not_called()
 
 
-tst_json_list = [{"name": "Test1", "html_url": "www.test1.com"},
-                 {"name": "Test2", "html_url": "www.test2.com"}]
-exp_components = [Component("Test1", source="www.test1.com"),
-                  Component("Test2", source="www.test2.com")]
-
-@pytest.mark.parametrize('json_list, exp_components',
-                         [(tst_json_list, exp_components)])
-def test_parse_json(json_list, exp_components):
+tst_json_list = [{"name": "Test1",
+                  "source": FAB_DEFS_URL,
+                  "html_url": "www.test1.com"},
+                 {"name": "Test2",
+                  "source": FAB_DEFS_URL,
+                  "html_url": "www.test2.com"}]
+exp_json_data = [{"name": "Test1",
+                  "source": FAB_DEFS_URL,
+                  "path": "www.test1.com"},
+                 {"name": "Test2",
+                  "source": FAB_DEFS_URL,
+                  "path": "www.test2.com"}]
+
+@pytest.mark.parametrize('json_list, exp_json_data',
+                         [(tst_json_list, exp_json_data)])
+def test_parse_json(json_list, exp_json_data):
     """Test parse_json function."""
-    assert parse_json(json_list) == exp_components
+    assert parse_json(json_list) == exp_json_data
+
+
+exp_new_components = [Component("Test1",
+                                source=FAB_DEFS_URL,
+                                path="www.test1.com"),
+                      Component("Test2",
+                                source=FAB_DEFS_URL,
+                                path="www.test2.com")]
+@pytest.mark.parametrize('tst_json_data, exp_components',
+                         [(exp_json_data, exp_new_components)])
+def test_construct_components(tst_json_data, exp_components):
+    """Test construct_components function."""
+    assert construct_components(tst_json_data) == exp_components
 
 
 tst_fab_comps = [Component("fabrikate-test-component"),