Repository: WIPACrepo/iceprod

Issue:
handle both named and positional args
For example, a script that is run like

```
./script.py --foo=bar baz
```

Discussion:

Patch:
diff --git a/iceprod/core/dataclasses.py b/iceprod/core/dataclasses.py
index 30452766..68bb380e 100644
--- a/iceprod/core/dataclasses.py
+++ b/iceprod/core/dataclasses.py
@@ -447,7 +447,6 @@ class Module(_TaskCommon):
             return (super(Module,self).valid() and
                     isinstance(self['running_class'],String) and
                     isinstance(self['src'],String) and
-                    isinstance(self['args'],String) and
                     isinstance(self['env_shell'],String) and
                     isinstance(self['env_clear'],bool) and
                     isinstance(self['configs'],dict)
diff --git a/iceprod/core/exe.py b/iceprod/core/exe.py
index db62233c..e71fa02c 100644
--- a/iceprod/core/exe.py
+++ b/iceprod/core/exe.py
@@ -876,10 +876,11 @@ class ForkModule:
         args = self.module['args']
         if args:
             self.logger.warning('args=%s',args)
-            if (args and isinstance(args,dataclasses.String) and
-                args[0] in ('{','[')):
+            if args and isinstance(args,dataclasses.String) and args[0] in ('{','['):
                 args = json_decode(args)
-            if isinstance(args,dataclasses.String):
+            if args and isinstance(args, dict) and set(args) == {'args','kwargs'}:
+                args = self.cfg.parseObject(args, self.env)
+            elif isinstance(args,dataclasses.String):
                 args = {"args":[self.cfg.parseValue(x,self.env) for x in args.split()],"kwargs":{}}
             elif isinstance(args,list):
                 args = {"args":[self.cfg.parseValue(x,self.env) for x in args],"kwargs":{}}
diff --git a/iceprod/server/ssl_cert.py b/iceprod/server/ssl_cert.py
index f5864fb2..776b4dcb 100644
--- a/iceprod/server/ssl_cert.py
+++ b/iceprod/server/ssl_cert.py
@@ -36,7 +36,7 @@ def create_ca(cert_filename,key_filename,days=365,hostname=None):
 
         # create a key pair
         k = crypto.PKey()
-        k.generate_key(crypto.TYPE_RSA, 2048)
+        k.generate_key(crypto.TYPE_RSA, 4096)
 
         # create a self-signed cert
         cert = crypto.X509()
@@ -81,7 +81,7 @@ def create_ca(cert_filename,key_filename,days=365,hostname=None):
                                  subject=cert),
             crypto.X509Extension(b"subjectAltName", False, b'DNS:'+hostname)
             ])
-        cert.sign(k, 'sha1')
+        cert.sign(k, 'sha512')
 
         open(cert_filename, "wb").write(
             crypto.dump_certificate(crypto.FILETYPE_PEM, cert))
@@ -143,7 +143,7 @@ def create_cert(cert_filename,key_filename,days=365,hostname=None,
                     #                     subject=cert),
                     ])
             cert.add_extensions(exts)
-            cert.sign(k, 'sha1')
+            cert.sign(k, 'sha512')
 
         else:
             cert.get_subject().C = "US"
@@ -155,7 +155,7 @@ def create_cert(cert_filename,key_filename,days=365,hostname=None,
 
             # finish cert req
             cert.set_pubkey(k)
-            cert.sign(k, 'sha1')
+            cert.sign(k, 'sha512')
 
             # load CA
             cacert = os.path.abspath(os.path.expandvars(cacert))
@@ -184,7 +184,7 @@ def create_cert(cert_filename,key_filename,days=365,hostname=None,
                     #                     subject=cert),
                     ])
             cert2.add_extensions(exts)
-            cert2.sign(ca_key, 'sha1')
+            cert2.sign(ca_key, 'sha512')
 
             # overwrite cert req with real cert
             cert = cert2
diff --git a/requirements.txt b/requirements.txt
index 1429e6e1..c0457b76 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,15 +4,15 @@
 #
 #    pip-compile
 #
-boto3==1.21.39
+boto3==1.24.5
     # via iceprod (setup.py)
-botocore==1.24.39
+botocore==1.27.5
     # via
     #   boto3
     #   s3transfer
-cachetools==5.0.0
+cachetools==5.2.0
     # via iceprod (setup.py)
-certifi==2021.10.8
+certifi==2022.5.18.1
     # via
     #   iceprod (setup.py)
     #   requests
@@ -20,7 +20,7 @@ cffi==1.15.0
     # via cryptography
 charset-normalizer==2.0.12
     # via requests
-cryptography==36.0.2
+cryptography==37.0.2
     # via
     #   iceprod (setup.py)
     #   pyopenssl
@@ -32,17 +32,17 @@ jmespath==1.0.0
     #   botocore
 ldap3==2.9.1
     # via iceprod (setup.py)
-motor==2.5.1
+motor==3.0.0
     # via iceprod (setup.py)
-psutil==5.9.0
+psutil==5.9.1
     # via iceprod (setup.py)
 pyasn1==0.4.8
     # via ldap3
 pycparser==2.21
     # via cffi
-pyjwt==2.3.0
+pyjwt==2.4.0
     # via wipac-rest-tools
-pymongo==3.12.3
+pymongo==4.1.1
     # via
     #   iceprod (setup.py)
     #   motor
@@ -67,9 +67,9 @@ requests-futures==1.0.0
     #   wipac-rest-tools
 requests-toolbelt==0.9.1
     # via iceprod (setup.py)
-s3transfer==0.5.2
+s3transfer==0.6.0
     # via boto3
-setproctitle==1.2.2
+setproctitle==1.2.3
     # via iceprod (setup.py)
 six==1.16.0
     # via python-dateutil
@@ -79,15 +79,17 @@ tornado==6.1
     # via
     #   iceprod (setup.py)
     #   wipac-rest-tools
+typing-extensions==4.2.0
+    # via wipac-dev-tools
 unidecode==1.3.4
     # via iceprod (setup.py)
 urllib3==1.26.9
     # via
     #   botocore
     #   requests
-wipac-dev-tools==1.1.5
+wipac-dev-tools==1.2.2
     # via
     #   iceprod (setup.py)
     #   wipac-rest-tools
-wipac-rest-tools==1.3.1
+wipac-rest-tools==1.3.3
     # via iceprod (setup.py)


Test patch:
diff --git a/tests/core/exe_test.py b/tests/core/exe_test.py
index 5da6a78c..636e57a2 100644
--- a/tests/core/exe_test.py
+++ b/tests/core/exe_test.py
@@ -1543,6 +1543,174 @@ cat foobar|grep 123
                 logger.error('running the module succeeded when not supposed to')
                 raise Exception('running the module succeeded when not supposed to')
 
+    @patch('iceprod.core.exe.functions.download')
+    @unittest_reporter(name='runmodule - args string')
+    async def test_214_runmodule_args(self, download):
+        # create the module object
+        module = iceprod.core.dataclasses.Module()
+        module['name'] = 'module'
+        module['src'] = 'file:/test.py'
+        module['args'] = '-a b --c d e'
+
+        # check that validate, resource_url, debug are in options
+        options = {}
+        if 'validate' not in options:
+            options['validate'] = True
+        if 'resource_url' not in options:
+            options['resource_url'] = 'http://foo/'
+        if 'debug' not in options:
+            options['debug'] = False
+
+        # make sure some basic options are set
+        if 'data_url' not in options:
+            options['data_url'] = 'gsiftp://gridftp/'
+        if 'svn_repository' not in options:
+            options['svn_repository'] = 'http://svn/'
+        if 'job_temp' not in options:
+            options['job_temp'] = os.path.join(self.test_dir,'job_temp')
+        if 'local_temp' not in options:
+            options['local_temp'] = os.path.join(self.test_dir,'local_temp')
+
+        async def create(*args, **kwargs):
+            path = os.path.join(options['local_temp'], os.path.basename(module['src']))
+            self.mk_files(path, """
+import argparse
+p = argparse.ArgumentParser()
+p.add_argument('-a')
+p.add_argument('--c')
+p.add_argument('e', nargs='+')
+args = p.parse_args()
+assert args.a == 'b'
+assert args.c == 'd'
+assert args.e == ['e']
+""", ext=True)
+            return path
+        download.side_effect = create
+
+        # set env
+        env = {'options': options}
+
+        # run the module
+        with to_log(sys.stdout,'stdout'),to_log(sys.stderr,'stderr'):
+            try:
+                async for mod in iceprod.core.exe.runmodule(self.config, env, module):
+                    await mod.wait()
+            except:
+                logger.error('running the module failed')
+                raise
+
+    @patch('iceprod.core.exe.functions.download')
+    @unittest_reporter(name='runmodule - args json')
+    async def test_215_runmodule_args(self, download):
+        # create the module object
+        module = iceprod.core.dataclasses.Module()
+        module['name'] = 'module'
+        module['src'] = 'file:/test.py'
+        module['args'] = '{"kwargs": {"a": "b", "cc": "d"}, "args": ["e"]}'
+
+        # check that validate, resource_url, debug are in options
+        options = {}
+        if 'validate' not in options:
+            options['validate'] = True
+        if 'resource_url' not in options:
+            options['resource_url'] = 'http://foo/'
+        if 'debug' not in options:
+            options['debug'] = False
+
+        # make sure some basic options are set
+        if 'data_url' not in options:
+            options['data_url'] = 'gsiftp://gridftp/'
+        if 'svn_repository' not in options:
+            options['svn_repository'] = 'http://svn/'
+        if 'job_temp' not in options:
+            options['job_temp'] = os.path.join(self.test_dir,'job_temp')
+        if 'local_temp' not in options:
+            options['local_temp'] = os.path.join(self.test_dir,'local_temp')
+
+        async def create(*args, **kwargs):
+            path = os.path.join(options['local_temp'], os.path.basename(module['src']))
+            self.mk_files(path, """
+import argparse
+p = argparse.ArgumentParser()
+p.add_argument('-a')
+p.add_argument('--cc')
+p.add_argument('e', nargs='+')
+args = p.parse_args()
+assert args.a == 'b'
+assert args.cc == 'd'
+assert args.e == ['e']
+""", ext=True)
+            return path
+        download.side_effect = create
+
+        # set env
+        env = {'options': options}
+
+        # run the module
+        with to_log(sys.stdout,'stdout'),to_log(sys.stderr,'stderr'):
+            try:
+                async for mod in iceprod.core.exe.runmodule(self.config, env, module):
+                    await mod.wait()
+            except:
+                logger.error('running the module failed')
+                raise
+
+    @patch('iceprod.core.exe.functions.download')
+    @unittest_reporter(name='runmodule - args dict')
+    async def test_216_runmodule_args(self, download):
+        # create the module object
+        module = iceprod.core.dataclasses.Module()
+        module['name'] = 'module'
+        module['src'] = 'file:/test.py'
+        module['args'] = {"kwargs": {"a": "b", "cc": "d"}, "args": ["e"]}
+
+        # check that validate, resource_url, debug are in options
+        options = {}
+        if 'validate' not in options:
+            options['validate'] = True
+        if 'resource_url' not in options:
+            options['resource_url'] = 'http://foo/'
+        if 'debug' not in options:
+            options['debug'] = False
+
+        # make sure some basic options are set
+        if 'data_url' not in options:
+            options['data_url'] = 'gsiftp://gridftp/'
+        if 'svn_repository' not in options:
+            options['svn_repository'] = 'http://svn/'
+        if 'job_temp' not in options:
+            options['job_temp'] = os.path.join(self.test_dir,'job_temp')
+        if 'local_temp' not in options:
+            options['local_temp'] = os.path.join(self.test_dir,'local_temp')
+
+        async def create(*args, **kwargs):
+            path = os.path.join(options['local_temp'], os.path.basename(module['src']))
+            self.mk_files(path, """
+import argparse
+p = argparse.ArgumentParser()
+p.add_argument('-a')
+p.add_argument('--cc')
+p.add_argument('e', nargs='+')
+args = p.parse_args()
+assert args.a == 'b'
+assert args.cc == 'd'
+assert args.e == ['e']
+""", ext=True)
+            return path
+        download.side_effect = create
+
+        # set env
+        env = {'options': options}
+
+        # run the module
+        with to_log(sys.stdout,'stdout'),to_log(sys.stderr,'stderr'):
+            try:
+                async for mod in iceprod.core.exe.runmodule(self.config, env, module):
+                    await mod.wait()
+            except:
+                logger.error('running the module failed')
+                raise
+
     @patch('iceprod.core.exe.functions.download')
     @unittest_reporter(name='runmodule - python script (clear env)')
     async def test_220_runmodule_script(self, download):