Repository: pex-tool/pex

Issue:
add a `-D` flag to pex(1) to support the ingestion of a structured dir of sources into an output pex
currently, pex(1) only operates against 3rdparty deps for composition of an output pex. in pants, we use pex as a library to create pex files that contain 3rdparty deps, sources and resources.

in order to help unblock utilization of pex(1) as a CLI tool in the v2 engine python pipeline porting, we'd like to add a flag to pex(1) to permit ingestion of a directory of python sources into a pex. this will differ from resources ingestion because sources need to be byte compiled by pex.

success criteria: can craft a pex via pex(1) that contains a package of importable python sources, with a callable entrypoint that can also be targeted by `-e` and successfully ran.


Discussion:

Resolving patch:
diff --git a/pex/bin/pex.py b/pex/bin/pex.py
--- a/pex/bin/pex.py
+++ b/pex/bin/pex.py
@@ -417,6 +417,26 @@ def configure_clp():
       type=str,
       help='The name of a file to be included as the preamble for the generated .pex file')
 
+  parser.add_option(
+      '-D', '--sources-directory',
+      dest='sources_directory',
+      metavar='DIR',
+      default=[],
+      type=str,
+      action='append',
+      help='Add sources directory to be packaged into the generated .pex file.'
+           '  This option can be used multiple times.')
+
+  parser.add_option(
+      '-R', '--resources-directory',
+      dest='resources_directory',
+      metavar='DIR',
+      default=[],
+      type=str,
+      action='append',
+      help='Add resources directory to be packaged into the generated .pex file.'
+           '  This option can be used multiple times.')
+
   parser.add_option(
       '-r', '--requirement',
       dest='requirement_files',
@@ -587,6 +607,20 @@ def build_pex(args, options, resolver_option_builder):
 
   pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble)
 
+  def walk_and_do(fn, src_dir):
+    src_dir = os.path.normpath(src_dir)
+    for root, dirs, files in os.walk(src_dir):
+      for f in files:
+        src_file_path = os.path.join(root, f)
+        dst_path = os.path.relpath(src_file_path, src_dir)
+        fn(src_file_path, dst_path)
+
+  for directory in options.sources_directory:
+    walk_and_do(pex_builder.add_source, directory)
+
+  for directory in options.resources_directory:
+    walk_and_do(pex_builder.add_resource, directory)
+
   pex_info = pex_builder.info
   pex_info.zip_safe = options.zip_safe
   pex_info.pex_path = options.pex_path


Test patch:
diff --git a/tests/test_integration.py b/tests/test_integration.py
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -909,3 +909,53 @@ def test_cross_platform_abi_targeting_behavior_exact():
                            'MarkupSafe==1.0',
                            '-o', pex_out_path])
     res.assert_success()
+
+
+def test_pex_source_bundling():
+  with temporary_dir() as output_dir:
+    with temporary_dir() as input_dir:
+      with open(os.path.join(input_dir, 'exe.py'), 'w') as fh:
+        fh.write(dedent('''
+          print('hello')
+          '''
+          ))
+
+      pex_path = os.path.join(output_dir, 'pex1.pex')
+      res = run_pex_command([
+        '-o', pex_path,
+        '-D', input_dir,
+        '-e', 'exe',
+      ])
+      res.assert_success()
+
+      stdout, rc = run_simple_pex(pex_path)
+
+      assert rc == 0
+      assert stdout == b'hello\n'
+
+
+def test_pex_resource_bundling():
+  with temporary_dir() as output_dir:
+    with temporary_dir() as input_dir, temporary_dir() as resources_input_dir:
+      with open(os.path.join(resources_input_dir, 'greeting'), 'w') as fh:
+        fh.write('hello')
+      pex_path = os.path.join(output_dir, 'pex1.pex')
+
+      with open(os.path.join(input_dir, 'exe.py'), 'w') as fh:
+        fh.write(dedent('''
+          import pkg_resources
+          print(pkg_resources.resource_string('__main__', 'greeting').decode('utf-8'))
+          '''))
+
+      res = run_pex_command([
+        '-o', pex_path,
+        '-D', input_dir,
+        '-R', resources_input_dir,
+        '-e', 'exe',
+      ])
+      res.assert_success()
+
+      stdout, rc = run_simple_pex(pex_path)
+
+      assert rc == 0
+      assert stdout == b'hello\n'