Repository: phoenixframework/phoenix
Pull request #4014 (2020-10-07 04:55:01)

Improve integration test speed
This improves integration test speed by following @josevalim's suggestions in #4012 (thanks José!). Below is an overview of the approach this takes:

1. Creates a new mix project in `integration_test`. This project contains the list of dependencies used by all phoenix projects, so the dependencies can be locked via `integration_test/mix.lock` and old builds can be re-run even if new versions of dependencies have been uploaded to hex.
2. To improve test speed of the generated projects, `mix.lock`, `deps`, and `_build` are copied from the integration test project to every generated project. Since some commands are run with `MIX_ENV=test` and others are run with `MIX_ENV=dev` by default, this initially compiles the dependencies in the test environment and then copies `_build/test` to `_build/dev`.
3. Since the integration test is a separate project now, `CodeGeneratorCase` is now being compiled at compile time instead of being required at runtime as a `.exs` file.

This brings the actual ex_unit test execution time from 80+ seconds to ~11 seconds.


josevalim: This looks great! :heart: I have added only two very minor comments. :)

Patch:
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a22513428..f63365c9c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -59,9 +59,12 @@ jobs:
           mix local.rebar --force
           mix local.hex --force
           apk add --no-progress --update git
-          mix deps.get
+      - name: Install dependencies
+        run: mix deps.get
+        working-directory: integration_test
       - name: Run integration tests
-        run: mix test.integration
+        run: mix test
+        working-directory: integration_test
 
   npm_test:
     runs-on: ubuntu-latest
diff --git a/.gitignore b/.gitignore
index b0e95da35..c4228d701 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,4 +11,7 @@
 /installer/doc
 /installer/deps
 
+/integration_test/_build
+/integration_test/deps
+
 erl_crash.dump
diff --git a/mix.exs b/mix.exs
index 655a2cc81..d3ed848b7 100644
--- a/mix.exs
+++ b/mix.exs
@@ -28,7 +28,6 @@ defmodule Phoenix.MixProject do
         ]
       ],
       elixirc_paths: elixirc_paths(Mix.env()),
-      test_paths: test_paths(System.get_env("TEST_SUITE")),
       name: "Phoenix",
       docs: docs(),
       aliases: aliases(),
@@ -195,8 +194,7 @@ defmodule Phoenix.MixProject do
 
   defp aliases do
     [
-      docs: ["docs", &generate_js_docs/1],
-      "test.integration": [&start_integration_test/1]
+      docs: ["docs", &generate_js_docs/1]
     ]
   end
 
@@ -204,26 +202,4 @@ defmodule Phoenix.MixProject do
     Mix.Task.run("app.start")
     System.cmd("npm", ["run", "docs"], cd: "assets")
   end
-
-  defp test_paths("integration"), do: ["integration_test"]
-  defp test_paths(_), do: ["test"]
-
-  defp start_integration_test(args) do
-    mix_cmd_with_status_check(
-      ["test", ansi_option() | args],
-      env: [{"TEST_SUITE", "integration"}]
-    )
-  end
-
-  defp ansi_option do
-    if IO.ANSI.enabled?(), do: "--color", else: "--no-color"
-  end
-
-  defp mix_cmd_with_status_check(args, opts) when is_list(opts) do
-    {_, res} = System.cmd("mix", args, [into: IO.binstream(:stdio, :line)] ++ opts)
-
-    if res > 0 do
-      System.at_exit(fn _ -> exit({:shutdown, 1}) end)
-    end
-  end
 end


Test patch:
diff --git a/integration_test/README.md b/integration_test/README.md
new file mode 100644
index 000000000..c1f1b3059
--- /dev/null
+++ b/integration_test/README.md
@@ -0,0 +1,25 @@
+## Phoenix Integration Tests
+
+This project contains integration tests for phoenix's generated projects.
+
+## Running tests
+
+To install dependencies, run:
+
+    $ mix deps.get
+
+Then run the tests with:
+
+    $ mix test
+
+## How tests are written
+
+In order to have consistent, repeatable builds, all dependencies for all phoenix
+project variations are listed in `mix.exs` and locked via `mix.lock`. If a
+dependency version needs to be updated, it can be updated with `mix.exs` or
+using `mix deps.update <dep name>`.
+
+It is also important to note that dependencies are initially compiled with
+`MIX_ENV=test` and then copied to `_build/dev_` to improve test speed.
+Therefore, dependencies should not be listed in `mix.exs` with an `only: <env>`
+option.
diff --git a/integration_test/mix.exs b/integration_test/mix.exs
new file mode 100644
index 000000000..92223f9a7
--- /dev/null
+++ b/integration_test/mix.exs
@@ -0,0 +1,46 @@
+defmodule Phoenix.Integration.MixProject do
+  use Mix.Project
+
+  def project do
+    [
+      app: :phoenix_integration,
+      version: "0.1.0",
+      elixir: "~> 1.9",
+      elixirc_paths: elixirc_paths(Mix.env()),
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  defp elixirc_paths(:test), do: ["lib", "test/support"]
+  defp elixirc_paths(_), do: ["lib"]
+
+  def application do
+    [
+      extra_applications: [:logger]
+    ]
+  end
+
+  # IMPORTANT: Dependencies are initially compiled with `MIX_ENV=test` and then
+  # copied to `_build/dev` to save time. Any dependencies with `only: :dev` set
+  # will not be copied.
+  defp deps do
+    [
+      {:phx_new, path: "../installer"},
+      {:phoenix, path: "../", override: true},
+      {:phoenix_ecto, "~> 4.1"},
+      {:ecto_sql, "~> 3.4.4"},
+      {:postgrex, ">= 0.0.0"},
+      {:phoenix_live_view, "~> 0.14.0"},
+      {:floki, ">= 0.27.0"},
+      {:phoenix_html, "~> 2.11"},
+      {:phoenix_live_reload, "~> 1.2"},
+      {:phoenix_live_dashboard, "~> 0.2"},
+      {:telemetry_metrics, "~> 0.4"},
+      {:telemetry_poller, "~> 0.4"},
+      {:gettext, "~> 0.11"},
+      {:jason, "~> 1.0"},
+      {:plug_cowboy, "~> 2.0"}
+    ]
+  end
+end
diff --git a/integration_test/mix.lock b/integration_test/mix.lock
new file mode 100644
index 000000000..6cf42d5c3
--- /dev/null
+++ b/integration_test/mix.lock
@@ -0,0 +1,29 @@
+%{
+  "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm", "4a0850c9be22a43af9920a71ab17c051f5f7d45c209e40269a1938832510e4d9"},
+  "cowboy": {:hex, :cowboy, "2.8.0", "f3dc62e35797ecd9ac1b50db74611193c29815401e53bac9a5c0577bd7bc667d", [:rebar3], [{:cowlib, "~> 2.9.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "4643e4fba74ac96d4d152c75803de6fad0b3fa5df354c71afdd6cbeeb15fac8a"},
+  "cowlib": {:hex, :cowlib, "2.9.1", "61a6c7c50cf07fdd24b2f45b89500bb93b6686579b069a89f88cb211e1125c78", [:rebar3], [], "hexpm", "e4175dc240a70d996156160891e1c62238ede1729e45740bdd38064dad476170"},
+  "db_connection": {:hex, :db_connection, "2.2.2", "3bbca41b199e1598245b716248964926303b5d4609ff065125ce98bcd368939e", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm", "642af240d8a8affb93b4ba5a6fcd2bbcbdc327e1a524b825d383711536f8070c"},
+  "decimal": {:hex, :decimal, "2.0.0", "a78296e617b0f5dd4c6caf57c714431347912ffb1d0842e998e9792b5642d697", [:mix], [], "hexpm", "34666e9c55dea81013e77d9d87370fe6cb6291d1ef32f46a1600230b1d44f577"},
+  "ecto": {:hex, :ecto, "3.4.6", "08f7afad3257d6eb8613309af31037e16c36808dfda5a3cd0cb4e9738db030e4", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6f13a9e2a62e75c2dcfc7207bfc65645ab387af8360db4c89fee8b5a4bf3f70b"},
+  "ecto_sql": {:hex, :ecto_sql, "3.4.5", "30161f81b167d561a9a2df4329c10ae05ff36eca7ccc84628f2c8b9fa1e43323", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "31990c6a3579b36a3c0841d34a94c275e727de8b84f58509da5f1b2032c98ac2"},
+  "file_system": {:hex, :file_system, "0.2.9", "545b9c9d502e8bfa71a5315fac2a923bd060fd9acb797fe6595f54b0f975fd32", [:mix], [], "hexpm", "3cf87a377fe1d93043adeec4889feacf594957226b4f19d5897096d6f61345d8"},
+  "floki": {:hex, :floki, "0.29.0", "b1710d8c93a2f860dc2d7adc390dd808dc2fb8f78ee562304457b75f4c640881", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "008585ce64b9f74c07d32958ec9866f4b8a124bf4da1e2941b28e41384edaaad"},
+  "gettext": {:hex, :gettext, "0.18.2", "7df3ea191bb56c0309c00a783334b288d08a879f53a7014341284635850a6e55", [:mix], [], "hexpm", "f9f537b13d4fdd30f3039d33cb80144c3aa1f8d9698e47d7bcbcc8df93b1f5c5"},
+  "html_entities": {:hex, :html_entities, "0.5.1", "1c9715058b42c35a2ab65edc5b36d0ea66dd083767bef6e3edb57870ef556549", [:mix], [], "hexpm", "30efab070904eb897ff05cd52fa61c1025d7f8ef3a9ca250bc4e6513d16c32de"},
+  "jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"},
+  "mime": {:hex, :mime, "1.4.0", "5066f14944b470286146047d2f73518cf5cca82f8e4815cf35d196b58cf07c47", [:mix], [], "hexpm", "75fa42c4228ea9a23f70f123c74ba7cece6a03b1fd474fe13f6a7a85c6ea4ff6"},
+  "phoenix_ecto": {:hex, :phoenix_ecto, "4.2.1", "13f124cf0a3ce0f1948cf24654c7b9f2347169ff75c1123f44674afee6af3b03", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 2.15", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "478a1bae899cac0a6e02be1deec7e2944b7754c04e7d4107fc5a517f877743c0"},
+  "phoenix_html": {:hex, :phoenix_html, "2.14.2", "b8a3899a72050f3f48a36430da507dd99caf0ac2d06c77529b1646964f3d563e", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "58061c8dfd25da5df1ea0ca47c972f161beb6c875cd293917045b92ffe1bf617"},
+  "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.2.9", "ac43a73561a4010fd2a52289a2f570829e2be5d5ea408a5af99dbed8793439e7", [:mix], [{:phoenix_html, "~> 2.14.1 or ~> 2.15", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.14.3", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.4.0 or ~> 0.5.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "f7cd3265a53d2bcd2e541a7d3c55f5a22e07bf6070b1d4adabd81791885190f7"},
+  "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.2.4", "940c0344b1d66a2e46eef02af3a70e0c5bb45a4db0bf47917add271b76cd3914", [:mix], [{:file_system, "~> 0.2.1 or ~> 0.3", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "38f9308357dea4cc77f247e216da99fcb0224e05ada1469167520bed4cb8cccd"},
+  "phoenix_live_view": {:hex, :phoenix_live_view, "0.14.7", "e05ca2e57974bb99eb54fed88b04754a622e54cf7e832db3c868bd06e0b99ff2", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.5.3", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 0.5", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "899224a704221ab0019200da61019dea699763e12daa24d69edd79bc228fe5a5"},
+  "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.0.0", "a1ae76717bb168cdeb10ec9d92d1480fec99e3080f011402c0a2d68d47395ffb", [:mix], [], "hexpm", "c52d948c4f261577b9c6fa804be91884b381a7f8f18450c5045975435350f771"},
+  "plug": {:hex, :plug, "1.10.4", "41eba7d1a2d671faaf531fa867645bd5a3dce0957d8e2a3f398ccff7d2ef017f", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ad1e233fe73d2eec56616568d260777b67f53148a999dc2d048f4eb9778fe4a0"},
+  "plug_cowboy": {:hex, :plug_cowboy, "2.3.0", "149a50e05cb73c12aad6506a371cd75750c0b19a32f81866e1a323dda9e0e99d", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "bc595a1870cef13f9c1e03df56d96804db7f702175e4ccacdb8fc75c02a7b97e"},
+  "plug_crypto": {:hex, :plug_crypto, "1.1.2", "bdd187572cc26dbd95b87136290425f2b580a116d3fb1f564216918c9730d227", [:mix], [], "hexpm", "6b8b608f895b6ffcfad49c37c7883e8df98ae19c6a28113b02aa1e9c5b22d6b5"},
+  "postgrex": {:hex, :postgrex, "0.15.6", "a464c72010a56e3214fe2b99c1a76faab4c2bb0255cabdef30dea763a3569aa2", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "f99268325ac8f66ffd6c4964faab9e70fbf721234ab2ad238c00f9530b8cdd55"},
+  "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm", "451d8527787df716d99dc36162fca05934915db0b6141bbdac2ea8d3c7afc7d7"},
+  "telemetry": {:hex, :telemetry, "0.4.2", "2808c992455e08d6177322f14d3bdb6b625fbcfd233a73505870d8738a2f4599", [:rebar3], [], "hexpm", "2d1419bd9dda6a206d7b5852179511722e2b18812310d304620c7bd92a13fcef"},
+  "telemetry_metrics": {:hex, :telemetry_metrics, "0.5.0", "1b796e74add83abf844e808564275dfb342bcc930b04c7577ab780e262b0d998", [:mix], [{:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "31225e6ce7a37a421a0a96ec55244386aec1c190b22578bd245188a4a33298fd"},
+  "telemetry_poller": {:hex, :telemetry_poller, "0.5.1", "21071cc2e536810bac5628b935521ff3e28f0303e770951158c73eaaa01e962a", [:rebar3], [{:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4cab72069210bc6e7a080cec9afffad1b33370149ed5d379b81c7c5f0c663fd4"},
+}
diff --git a/integration_test/code_generation/app_with_defaults_test.exs b/integration_test/test/code_generation/app_with_defaults_test.exs
similarity index 86%
rename from integration_test/code_generation/app_with_defaults_test.exs
rename to integration_test/test/code_generation/app_with_defaults_test.exs
index 3fcb64be4..f6a3a7995 100644
--- a/integration_test/code_generation/app_with_defaults_test.exs
+++ b/integration_test/test/code_generation/app_with_defaults_test.exs
@@ -1,5 +1,3 @@
-Code.require_file("../support/code_generator_case.exs", __DIR__)
-
 defmodule Phoenix.Integration.CodeGeneration.AppWithDefaultsTest do
   use Phoenix.Integration.CodeGeneratorCase, async: true
 
diff --git a/integration_test/code_generation/app_with_no_options_test.exs b/integration_test/test/code_generation/app_with_no_options_test.exs
similarity index 89%
rename from integration_test/code_generation/app_with_no_options_test.exs
rename to integration_test/test/code_generation/app_with_no_options_test.exs
index 65858f505..34094a02b 100644
--- a/integration_test/code_generation/app_with_no_options_test.exs
+++ b/integration_test/test/code_generation/app_with_no_options_test.exs
@@ -1,5 +1,3 @@
-Code.require_file("../support/code_generator_case.exs", __DIR__)
-
 defmodule Phoenix.Integration.CodeGeneration.AppWithNoOptionsTest do
   use Phoenix.Integration.CodeGeneratorCase, async: true
 
diff --git a/integration_test/code_generation/umbrella_app_with_defaults_test.exs b/integration_test/test/code_generation/umbrella_app_with_defaults_test.exs
similarity index 86%
rename from integration_test/code_generation/umbrella_app_with_defaults_test.exs
rename to integration_test/test/code_generation/umbrella_app_with_defaults_test.exs
index 6fcb20f21..d1eafa4a5 100644
--- a/integration_test/code_generation/umbrella_app_with_defaults_test.exs
+++ b/integration_test/test/code_generation/umbrella_app_with_defaults_test.exs
@@ -1,5 +1,3 @@
-Code.require_file("../support/code_generator_case.exs", __DIR__)
-
 defmodule Phoenix.Integration.CodeGeneration.UmbrellaAppWithDefaultsTest do
   use Phoenix.Integration.CodeGeneratorCase, async: true
 
diff --git a/integration_test/support/code_generator_case.exs b/integration_test/test/support/code_generator_case.ex
similarity index 87%
rename from integration_test/support/code_generator_case.exs
rename to integration_test/test/support/code_generator_case.ex
index 0580e1fae..d99443909 100644
--- a/integration_test/support/code_generator_case.exs
+++ b/integration_test/test/support/code_generator_case.ex
@@ -10,12 +10,17 @@ defmodule Phoenix.Integration.CodeGeneratorCase do
   def generate_phoenix_app(tmp_dir, app_name, opts \\ [])
       when is_binary(app_name) and is_list(opts) do
     app_path = Path.expand(app_name, tmp_dir)
-    installer_root = Path.expand("../../installer", __DIR__)
+    integration_test_root_path = Path.expand("../../", __DIR__)
     app_root_path = get_app_root_path(tmp_dir, app_name, opts)
 
-    mix_run!(["phx.new", app_path, "--dev", "--no-install"] ++ opts, installer_root)
+    mix_run!(["phx.new", app_path, "--dev", "--no-install"] ++ opts, integration_test_root_path)
 
-    mix_run!(~w(deps.get), app_root_path)
+    for path <- ~w(mix.lock deps _build) do
+      File.cp_r!(
+        Path.join(integration_test_root_path, path),
+        Path.join(app_root_path, path)
+      )
+    end
 
     app_root_path
   end
@@ -69,7 +74,7 @@ defmodule Phoenix.Integration.CodeGeneratorCase do
   end
 
   defp installer_tmp_path do
-    Path.expand("../../installer/tmp", __DIR__)
+    Path.expand("../../../installer/tmp", __DIR__)
   end
 
   defp get_app_root_path(tmp_dir, app_name, opts) do
diff --git a/integration_test/test/test_helper.exs b/integration_test/test/test_helper.exs
new file mode 100644
index 000000000..837fc8c8b
--- /dev/null
+++ b/integration_test/test/test_helper.exs
@@ -0,0 +1,10 @@
+# Copy _build/test to _build/dev so it only has to be compiled once
+File.rm_rf!(Path.expand("../_build/dev", __DIR__))
+
+File.cp_r!(
+  Path.expand("../_build/test", __DIR__),
+  Path.expand("../_build/dev", __DIR__)
+)
+
+ExUnit.configure(timeout: 180_000)
+ExUnit.start()
diff --git a/integration_test/test_helper.exs b/integration_test/test_helper.exs
deleted file mode 100644
index 141158fc2..000000000
--- a/integration_test/test_helper.exs
+++ /dev/null
@@ -1,8 +0,0 @@
-Code.require_file("./support/code_generator_case.exs", __DIR__)
-
-# Compile installer app up front so multiple test cases
-# don't try to compile it at the same time.
-Phoenix.Integration.CodeGeneratorCase.mix_run!(["do", "deps.get,", "compile"], "./installer")
-
-ExUnit.configure(max_cases: 2, timeout: 180_000)
-ExUnit.start()