Repository: facebookincubator/ptr

Issue:
Add Support for Classes and Custom Commands in seutp.py AST Parsing
Thanks for logging an issue with `ptr`. Here are some questions to help try 
and pull out as much information possible to help us help you!

## How is `ptr` not doing what you expect?
Unable to run `ptr` on a project where `setup.py` includes a custom build command definition.

## What is your suggestion to make this better?
Add AST parsing support for custom commands and other class definitions.

## Code/Bug example?
```
root@faa971b550dd:/usr/local/tglib# ptr -k
[2019-08-06 19:50:50,751] INFO: Starting /usr/local/bin/ptr (ptr.py:1073)
Traceback (most recent call last):
  File "/usr/local/bin/ptr", line 10, in <module>
    sys.exit(main())
  File "/usr/local/lib/python3.7/dist-packages/ptr.py", line 1090, in main
    args.venv_timeout,
  File "/usr/lib/python3.7/asyncio/base_events.py", line 584, in run_until_complete
    return future.result()
  File "/usr/local/lib/python3.7/dist-packages/ptr.py", line 965, in async_main
    base_path, stats, run_disabled, print_non_configured
  File "/usr/local/lib/python3.7/dist-packages/ptr.py", line 337, in _get_test_modules
    if target.id == "ptr_params":  # type: ignore
```

## How can a developer reproduce this?

Add a custom command to `setup.py` using the instructions [here](https://jichu4n.com/posts/how-to-add-custom-build-steps-and-commands-to-setuppy/).

Comments:

Patch:
diff --git a/ptr.py b/ptr.py
index a6172a1..480b52b 100755
--- a/ptr.py
+++ b/ptr.py
@@ -303,8 +303,11 @@ def _parse_setup_params(setup_py: Path) -> Dict[str, Any]:
     for node in ast.walk(setup_tree):
         if isinstance(node, ast.Assign):
             for target in node.targets:
-                # mypy error: "expr" has no attribute "id"
-                if target.id == "ptr_params":  # type: ignore
+                target_id = getattr(target, "id", None)
+                if not target_id:
+                    continue
+
+                if target_id == "ptr_params":
                     LOG.debug("Found ptr_params in {}".format(setup_py))
                     ptr_params = dict(ast.literal_eval(node.value))
                     if "test_suite" in ptr_params:
@@ -427,20 +430,6 @@ async def _progress_reporter(
     LOG.debug("progress_reporter finished")
 
 
-# TODO: Remove when black works with Python 3.7 on Windows
-def _run_black(config: Dict, force_black: bool = False) -> bool:
-    """ Check if we're Windows and Python 3.7 or disabled via config
-       `black.exe` is broken with Python 3.7 - Issue Open:
-       https://github.com/ambv/black/issues/728 """
-    if not force_black and WINDOWS and sys.version_info >= (3, 7):
-        LOG.error(
-            "black is hard disabled due to not working with Python 3.7 on Windows"
-        )
-        return False
-
-    return bool("run_black" in config and config["run_black"])
-
-
 def _set_build_env(build_base_path: Optional[Path]) -> Dict[str, str]:
     build_environ = environ.copy()
     if not build_base_path or not build_base_path.exists():
@@ -496,7 +485,6 @@ async def _test_steps_runner(
     env: Dict,
     stats: Dict[str, int],
     print_cov: bool = False,
-    force_black: bool = False,
 ) -> Tuple[Optional[test_result], int]:
     bin_dir = "Scripts" if WINDOWS else "bin"
     exe = ".exe" if WINDOWS else ""
@@ -544,7 +532,7 @@ async def _test_steps_runner(
         ),
         step(
             StepName.black_run,
-            _run_black(config, force_black),
+            bool("run_black" in config and config["run_black"]),
             _generate_black_cmd(setup_py_path.parent, black_exe),
             "Running black for {}".format(setup_py_path),
             config["test_suite_timeout"],
@@ -648,7 +636,6 @@ async def _test_runner(
     test_results: List[test_result],
     venv_path: Path,
     print_cov: bool,
-    force_black: bool,
     stats: Dict[str, int],
     idx: int,
 ) -> None:
@@ -681,7 +668,6 @@ async def _test_runner(
             env,
             stats,
             print_cov,
-            force_black,
         )
         total_success_runtime = int(time() - test_run_start_time)
         if test_fail_result:
@@ -882,7 +868,6 @@ async def run_tests(
     venv_path: Optional[Path],
     venv_keep: bool,
     print_cov: bool,
-    force_black: bool,
     stats: Dict[str, int],
     stats_file: str,
     venv_timeout: float,
@@ -911,14 +896,7 @@ async def run_tests(
     test_results = []  # type: List[test_result]
     consumers = [
         _test_runner(
-            queue,
-            tests_to_run,
-            test_results,
-            venv_path,
-            print_cov,
-            force_black,
-            stats,
-            i + 1,
+            queue, tests_to_run, test_results, venv_path, print_cov, stats, i + 1
         )
         for i in range(atonce)
     ]
@@ -956,7 +934,6 @@ async def async_main(
     print_cov: bool,
     print_non_configured: bool,
     run_disabled: bool,
-    force_black: bool,
     stats_file: str,
     venv_timeout: float,
 ) -> int:
@@ -991,7 +968,6 @@ async def async_main(
         venv_path,
         venv_keep,
         print_cov,
-        force_black,
         stats,
         stats_file,
         venv_timeout,
@@ -1019,11 +995,6 @@ def main() -> None:
     parser.add_argument(
         "-d", "--debug", action="store_true", help="Verbose debug output"
     )
-    parser.add_argument(
-        "--force-black",
-        action="store_true",
-        help="Ensure black runs if enabled in config",
-    )
     parser.add_argument(
         "-k", "--keep-venv", action="store_true", help="Do not remove created venv"
     )
@@ -1086,7 +1057,6 @@ def main() -> None:
                     args.print_cov,
                     args.print_non_configured,
                     args.run_disabled,
-                    args.force_black,
                     args.stats_file,
                     args.venv_timeout,
                 )
diff --git a/setup.py b/setup.py
index c0c56d0..5a21d72 100644
--- a/setup.py
+++ b/setup.py
@@ -20,7 +20,7 @@ ptr_params = {
     "test_suite": "ptr_tests",
     "test_suite_timeout": 120,
     # Relative path from setup.py to module (e.g. ptr == ptr.py)
-    "required_coverage": {"ptr.py": 83, "TOTAL": 89},
+    "required_coverage": {"ptr.py": 84, "TOTAL": 89},
     # Run black or not
     "run_black": True,
     # Run mypy or not
@@ -47,7 +47,7 @@ def get_long_desc() -> str:
 
 setup(
     name=ptr_params["entry_point_module"],
-    version="19.7.16",
+    version="19.8.7",
     description="Parallel asyncio Python setup.(cfg|py) Test Runner",
     long_description=get_long_desc(),
     long_description_content_type="text/markdown",


Test patch:
diff --git a/ptr_tests.py b/ptr_tests.py
index fb8e02a..8d0bc00 100644
--- a/ptr_tests.py
+++ b/ptr_tests.py
@@ -13,7 +13,6 @@ from os import environ
 from pathlib import Path
 from shutil import rmtree
 from subprocess import CalledProcessError
-from sys import version_info
 from tempfile import TemporaryDirectory, gettempdir
 from typing import (  # noqa: F401 # pylint: disable=unused-import
     Any,
@@ -130,20 +129,7 @@ class TestPtr(unittest.TestCase):
     @patch("ptr.run_tests", async_none)
     @patch("ptr._get_test_modules")
     def test_async_main(self, mock_gtm: Mock) -> None:
-        args = [
-            1,
-            Path("/"),
-            "mirror",
-            1,
-            "venv",
-            True,
-            True,
-            False,
-            True,
-            False,
-            "stats",
-            30,
-        ]
+        args = [1, Path("/"), "mirror", 1, "venv", True, True, False, True, "stats", 30]
         mock_gtm.return_value = False
         self.assertEqual(
             self.loop.run_until_complete(ptr.async_main(*args)), 1  # pyre-ignore
@@ -382,17 +368,6 @@ class TestPtr(unittest.TestCase):
         )
         self.assertEqual(mock_log.call_count, 2)
 
-    def test_run_black(self) -> None:
-        config = {}  # type: Dict[str, Any]
-        self.assertFalse(ptr._run_black(config, False))
-        self.assertFalse(ptr._run_black(config, True))
-        config["run_black"] = True
-        if ptr.WINDOWS and version_info >= (3, 7):
-            # Ensure even if in config we don't run it
-            self.assertFalse(ptr._run_black(config, False))
-        else:
-            self.assertTrue(ptr._run_black(config, False))
-
     def test_set_build_env(self) -> None:
         local_build_path = Path(gettempdir())
         build_env = ptr._set_build_env(local_build_path)
@@ -429,7 +404,7 @@ class TestPtr(unittest.TestCase):
             stats = defaultdict(int)  # type: Dict[str, int]
             self.loop.run_until_complete(
                 ptr._test_runner(
-                    queue, tests_to_run, test_results, td_path, False, False, stats, 69
+                    queue, tests_to_run, test_results, td_path, False, stats, 69
                 )
             )
             self.assertEqual(len(test_results), 1)
@@ -464,14 +439,11 @@ class TestPtr(unittest.TestCase):
                 (None, 6) if ptr.WINDOWS else (None, 7),
             )
 
-            if ptr.WINDOWS and version_info >= (3, 7):
-                # No pyre + black
-                expected = (None, 6)
-            elif ptr.WINDOWS:
+            if ptr.WINDOWS:
                 # No pyre
                 expected = (None, 7)
             else:
-                # Running all steps on everything else
+                # Running all steps on all other platforms
                 expected = (None, 8)
 
             fake_tests_to_run = {fake_setup_py: ptr_tests_fixtures.EXPECTED_TEST_PARAMS}
diff --git a/ptr_tests_fixtures.py b/ptr_tests_fixtures.py
index cb51eaa..d278888 100644
--- a/ptr_tests_fixtures.py
+++ b/ptr_tests_fixtures.py
@@ -29,7 +29,7 @@ EXPECTED_TEST_PARAMS = {
     "entry_point_module": "ptr",
     "test_suite": "ptr_tests",
     "test_suite_timeout": 120,
-    "required_coverage": {"ptr.py": 83, "TOTAL": 89},
+    "required_coverage": {"ptr.py": 84, "TOTAL": 89},
     "run_black": True,
     "run_mypy": True,
     "run_flake8": True,
@@ -196,7 +196,7 @@ disabled = true
 entry_point_module = ptr
 test_suite = ptr_tests
 test_suite_timeout = 120
-required_coverage_ptr.py = 83
+required_coverage_ptr.py = 84
 required_coverage_TOTAL = 89
 run_black = true
 run_mypy = true