Repository: globocom/auror-core

Issue:
Read flow v2 from files to Python memory
Reading flow files and write Python code should be useful for Auror adoption.

It should just read yaml files and load to memory

Comments:

Patch:
diff --git a/.travis.yml b/.travis.yml
index 3890b73..c6a1bcb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,7 @@ python:
 
 # command to install dependencies
 install:
-  - pip install .
+  - pip install .[test]
 # command to run tests
 script:
   - make test
diff --git a/README.md b/README.md
index 0d32de5..9a58d1f 100644
--- a/README.md
+++ b/README.md
@@ -11,8 +11,32 @@
 pip install auror_core
 ```
 
+## Supported Job Types
+
+### V1
+
+* **Command**
+* **Flow**
+
+### V2
+
+* **Command**
+
 ## Usage
 
+* [Creating a simple Azkaban flow with one command](#creating-a-simple-azkaban-flow-with-one-command)
+* [Creating a simple V2 Azkaban flow with one command](#Creating-a-simple-v2-azkaban-flow-with-one-command )
+* [Creating Flows with dependencies](#creating-flows-with-dependencies)
+* [Sharing job attributes](#sharing-job-attributes)
+* [Job with extra customization and configuration](#job-with-extra-customization-and-configuration)
+* [Integrating with Flow (just for V1)](#integrating-with-flow-(just-for-v1))
+* [Using Flow Params](#using-flow-params)
+* [Using Flow Environment Variables](#using-flow-environment-variables)
+* [Using Flow Environment Variables and Params](#using-flow-environment-variables-and-params)
+* [Join multiple variables in one](#join-multiple-variables-in-one)
+* [Load jobs from YAML File (just for V2)](#Load-jobs-from-yaml-file-(just-for-v2))
+
+
 ### Creating a simple Azkaban flow with one command 
 
 You just need to import job type and project
@@ -234,6 +258,17 @@ Project("folder_to_generate_files", com)\
 
 ```
 
+### Load jobs from YAML File (just for V2)
+
+**You can find some YAML File examples on [Azkaban Flow Documentation](https://github.com/azkaban/azkaban/wiki/Azkaban-Flow-2.0-Design#flow-yaml-file)**
+
+```python
+from auror_core.v2.loader import Loader
+
+loader = Loader('/path/to/file/flow.yaml')
+jobs = loader.as_job_objects()
+```
+
 ## Plugins
 
 Plugins are just extensions from auror_core
diff --git a/auror_core/v2/__init__.py b/auror_core/v2/__init__.py
index e69de29..f5117c3 100644
--- a/auror_core/v2/__init__.py
+++ b/auror_core/v2/__init__.py
@@ -0,0 +1,9 @@
+from auror_core.v2.job import Command
+
+
+class JobType:
+    COMMAND = Command
+
+    @staticmethod
+    def get_job_type_class(job_type):
+        return getattr(JobType, job_type.upper(), Command)
diff --git a/auror_core/v2/job.py b/auror_core/v2/job.py
index 23ef111..6fea4b0 100644
--- a/auror_core/v2/job.py
+++ b/auror_core/v2/job.py
@@ -1,7 +1,12 @@
 import os
 import copy
+
 import yaml
 
+from collections import namedtuple
+from functools import reduce
+
+
 class Job(object):
 
     def __init__(self, name="DefaultJob", config=None, dependencies=None, nodes=None, extra=None):
@@ -11,6 +16,15 @@ class Job(object):
         self.nodes = nodes or []
         self.extra = extra or {}
         self.properties = dict(nodes=list())
+    
+    def __eq__(self, other):
+        return (isinstance(other, Job)) and \
+            self.name == other.name and \
+            self.config == other.config and \
+            self.dependencies == other.dependencies and \
+            self.nodes == other.nodes and \
+            self.extra == other.extra and \
+            self.properties == other.properties
 
     def instance(self, name, config, dependencies, nodes, extra):
         return self.__class__(name, config, dependencies, nodes, extra)
@@ -29,7 +43,7 @@ class Job(object):
         return self.instance(self.name, self.config, dependencies, self.nodes, self.extra)
 
     def with_nodes(self, *nodes):
-        return self.instance(self.name, self.config, self.dependencies, nodes, self.extra)
+        return self.instance(self.name, self.config, self.dependencies, list(nodes), self.extra)
 
     def with_(self, **extra):
         self_extra = copy.deepcopy(self.extra)
@@ -72,23 +86,60 @@ class Job(object):
     def _add_items(self):
         job = self.before_add_hook()
         self.properties["nodes"].append(self._get_node(job))
-
+    
+    @classmethod
+    def build(cls, data):
+        raise NotImplementedError('"build" method is not implemented')
 
 
 class Command(Job):
     _type = "command"
+    _Command = namedtuple('_Command', ['command', 'command_number'])
 
+    def __eq__(self, other):
+        return super(Command, self).__eq__(other) and \
+            isinstance(other, Command)
+            
     def with_all_default(self):
         return self.instance(self.name, self.config, self.dependencies, self.nodes, self.extra)
 
     def with_command(self, command):
         return self.with_(command=command)
-
-    def with_another_command(self, command):
+    
+    def with_another_commands(self, commands):
+        return reduce(
+            lambda instance, command: instance.with_another_command(*command),
+            commands,
+            self
+        )
+
+    def with_another_command(self, command, command_number=None):
+        if not command:
+            return self
+        
         if not self.extra.get("command"):
             return self.with_command(command)
-
+ 
+        command_number = command_number or self.__get_next_command_number()
+        return self.with_(**{"command.{}".format(command_number): command})
+    
+    def __get_next_command_number(self):
         counter = 1
         while self.extra.get("command.{}".format(counter)):
             counter += 1
-        return self.with_(**{"command.{}".format(counter): command})
+        return counter
+    
+    @classmethod
+    def build(cls, data):
+        extra_commands = [
+            cls._Command(value, key.split('.')[-1])
+            for key, value in data['config'].items() if 'command.' in key
+        ]
+        return cls(
+            name=data['name'],
+            config=data.get('config'),
+            dependencies=data.get('dependsOn'),
+            nodes=data.get('nodes'),
+            extra=data.get('extra')
+        ).with_command(data['config']['command']) \
+        .with_another_commands(extra_commands)
diff --git a/auror_core/v2/loader.py b/auror_core/v2/loader.py
new file mode 100644
index 0000000..9185af2
--- /dev/null
+++ b/auror_core/v2/loader.py
@@ -0,0 +1,34 @@
+import os
+import copy
+
+import yaml
+
+from auror_core.v2.job import Job
+from auror_core.v2 import JobType
+
+
+class Loader:
+    def __init__(self, flow_file_path):
+        if not os.path.exists(flow_file_path):
+            raise ValueError('File does not exists')
+        
+        self._config, self._jobs = self.__load_yaml(flow_file_path)
+    
+    def __load_yaml(self, path):
+        yaml_file = yaml.safe_load(open(path))
+        config = yaml_file['config'] if 'config' in yaml_file.keys() else {}
+        return config, yaml_file['nodes']
+    
+    def as_job_objects(self):
+        return self.__as_job_objects(self._jobs)
+    
+    def __as_job_objects(self, jobs):
+        return [self.__build_job(job) for job in jobs]
+    
+    def __build_job(self, job):
+        if job.get('nodes'):
+            job['nodes'] = self.__as_job_objects(job.get('nodes'))
+        return JobType.get_job_type_class(job.get('type')).build(job)
+
+    def as_python_file(self):
+        raise NotImplementedError('"as_python_file" method is not implemented yet')
diff --git a/setup.py b/setup.py
index b29f4ca..e998efc 100644
--- a/setup.py
+++ b/setup.py
@@ -17,5 +17,8 @@ setup(
         "javaproperties==0.5.1",
         "pyaml==18.11.0"
     ],
+    extras_require={
+        "test": ["mock",]
+    },
     packages=find_packages(),
 )


Test patch:
diff --git a/tests/unit/v2/test_command_job.py b/tests/unit/v2/test_command_job.py
new file mode 100644
index 0000000..8240e0b
--- /dev/null
+++ b/tests/unit/v2/test_command_job.py
@@ -0,0 +1,81 @@
+from unittest import TestCase
+
+from auror_core.v2.job import Job, Command
+
+
+class CommandJobTest(TestCase):
+
+    def test_with_command(self):
+        command = "${python} -c 'from teste import teste_command_spark; teste_command_spark()'"
+        result = Command().with_command(command=command)
+        actual = result.extra
+        expected = {'command': "${python} -c 'from teste import teste_command_spark; teste_command_spark()'"}
+
+        self.assertEqual('command', result._type)
+        self.assertEqual(expected, actual)
+
+    def test_with_another_command_without_a_command(self):
+        command = "${python} -c 'from teste import teste_command_spark; teste_command_spark()'"
+        result = Command().with_another_command(command=command)
+        actual = result.extra
+        expected = {'command': "${python} -c 'from teste import teste_command_spark; teste_command_spark()'"}
+
+        self.assertEqual('command', result._type)
+        self.assertEqual(expected, actual)
+
+    def test_with_another_command_where_a_command_already_exists(self):
+        command_ = "${python} -c 'from teste import teste_command_spark; teste_command_spark()'"
+        result_ = Command().with_another_command(command=command_)
+        command = "${python} -c 'from teste import teste_command_spark_again; teste_command_spark_again()'"
+        result = result_.with_another_command(command=command)
+        actual = result.extra
+        expected = {'command': "${python} -c 'from teste import teste_command_spark; teste_command_spark()'", 
+                    'command.1': "${python} -c 'from teste import teste_command_spark_again; teste_command_spark_again()'"}
+
+        self.assertEqual('command', result._type)
+        self.assertEqual(expected, actual)
+    
+    def test_build_method_with_one_command(self):
+        simple_job = {
+            'config': {
+                'command': 'COMMAND'
+            },
+            'dependsOn': ['firstDependencie', 'secondDependecie'],
+            'name': 'AZTest',
+        }
+        actual_job = Command.build(simple_job)
+
+        expected_job = Command(
+            simple_job['name'],
+            simple_job['config'],
+            simple_job['dependsOn']
+        ).with_command(simple_job['config']['command'])
+
+        self.assertEqual(expected_job, actual_job)
+    
+    def test_build_method_with_more_than_one_command(self):
+        extra_commands = {
+            'command.1': 'COMMAND 2',
+            'command.2': 'COMMAND 3',
+        }
+        job = {
+            'config': {
+                'command': 'COMMAND'
+            },
+            'dependsOn': ['firstDependencie', 'secondDependecie'],
+            'name': 'AZTest',
+        }
+        job['config'].update(extra_commands)
+        actual_job = Command.build(job)
+
+        expected_job = Command(
+            job['name'],
+            job['config'],
+            job['dependsOn']
+        ).with_command(job['config']['command']) \
+        .with_another_commands([
+            Command._Command(command, command_number.split('.')[-1])
+            for command_number, command in extra_commands.items()
+        ])
+
+        self.assertEqual(expected_job, actual_job)
diff --git a/tests/unit/v2/test_job.py b/tests/unit/v2/test_job.py
index 505f67b..0d07770 100644
--- a/tests/unit/v2/test_job.py
+++ b/tests/unit/v2/test_job.py
@@ -1,9 +1,12 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+import shutil
+import tempfile
+
 from os import path
-import shutil, tempfile
 from unittest import TestCase
+
 from auror_core.v2.job import Job, Command
 
 
@@ -87,42 +90,3 @@ class JobTest(TestCase):
 
         self.assertEqual(None, content.properties["nodes"][0].get("dependsOn"))
         self.assertEqual("command", content.properties["nodes"][0]["type"])
-
-
-class CommandJobTest(TestCase):
-
-    def test_with_all_default(self):
-        expected = Job("test_job_name", None, ["other_job_name"], None,
-                            {"executor.cores": "2", "driver.memory": "6g"})
-
-        self.assertIsInstance(expected, Job)
-
-    def test_with_command(self):
-        command = "${python} -c 'from teste import teste_command_spark; teste_command_spark()'"
-        result = Command().with_command(command=command)
-        actual = result.extra
-        expected = {'command': "${python} -c 'from teste import teste_command_spark; teste_command_spark()'"}
-
-        self.assertEqual('command', result._type)
-        self.assertEqual(expected, actual)
-
-    def test_with_another_command_without_a_command(self):
-        command = "${python} -c 'from teste import teste_command_spark; teste_command_spark()'"
-        result = Command().with_another_command(command=command)
-        actual = result.extra
-        expected = {'command': "${python} -c 'from teste import teste_command_spark; teste_command_spark()'"}
-
-        self.assertEqual('command', result._type)
-        self.assertEqual(expected, actual)
-
-    def test_with_another_command_where_a_command_already_exists(self):
-        command_ = "${python} -c 'from teste import teste_command_spark; teste_command_spark()'"
-        result_ = Command().with_another_command(command=command_)
-        command = "${python} -c 'from teste import teste_command_spark_again; teste_command_spark_again()'"
-        result = result_.with_another_command(command=command)
-        actual = result.extra
-        expected = {'command': "${python} -c 'from teste import teste_command_spark; teste_command_spark()'", 
-                    'command.1': "${python} -c 'from teste import teste_command_spark_again; teste_command_spark_again()'"}
-
-        self.assertEqual('command', result._type)
-        self.assertEqual(expected, actual)
\ No newline at end of file
diff --git a/tests/unit/v2/test_loader.py b/tests/unit/v2/test_loader.py
new file mode 100644
index 0000000..26b2479
--- /dev/null
+++ b/tests/unit/v2/test_loader.py
@@ -0,0 +1,69 @@
+import shutil
+import tempfile
+import os
+
+import mock
+
+from unittest import TestCase
+
+from auror_core.v2.job import Job, Command
+from auror_core.v2.loader import Loader
+
+
+class LoaderTest(TestCase):
+
+    def test_error_on_inexistence_file_path(self):
+        with self.assertRaises(ValueError) as context:
+            Loader('/this/does/not/exist/flow.yaml')
+        self.assertTrue('File does not exists' in str(context.exception))
+    
+    @mock.patch('auror_core.v2.loader.os')
+    @mock.patch('auror_core.v2.loader.yaml')
+    @mock.patch('auror_core.v2.loader.open')
+    def test_should_return_command_type_job_list(self, mock_os, mock_yaml, mock_open):
+        job = {
+            'config': {
+                'command': 'COMMAND'
+            },
+            'dependsOn': ['firstDependencie', 'secondDependecie'],
+            'name': 'AZTest',
+            'type': 'command'
+        }
+        mock_os.path.exists.return_value = True
+        mock_yaml.safe_load.return_value = {'nodes': [job, job]}
+
+        loader = Loader('/flow/path/flow.yaml')
+        jobs = loader.as_job_objects()
+
+        self.assertIsInstance(jobs, list)
+        self.assertEqual(2, len(jobs))
+        self.assertTrue(all([isinstance(job, Job) for job in jobs]))
+    
+    @mock.patch('auror_core.v2.loader.os')
+    @mock.patch('auror_core.v2.loader.yaml')
+    @mock.patch('auror_core.v2.loader.open')
+    def test_should_return_embbed_flow(self, mock_os, mock_yaml, mock_open):
+        job = {
+            'config': {
+                'command': 'COMMAND'
+            },
+            'dependsOn': ['firstDependencie', 'secondDependecie'],
+            'name': 'AZTest',
+            'type': 'command'
+        }
+        job['nodes'] = [job.copy(),]
+        mock_os.path.exists.return_value = True
+        mock_yaml.safe_load.return_value = {'nodes': [job,]}
+
+        loader = Loader('/flow/path/flow.yaml')
+        jobs = loader.as_job_objects()
+
+        expected_job = Command(
+            job['name'],
+            job['config'],
+            job['dependsOn'],
+        ).with_command(job['config']['command'])
+        expected_job = expected_job.with_nodes(expected_job)
+
+        self.assertEqual(1, len(jobs))
+        self.assertEqual(expected_job, jobs[0])