Repository: derekkraan/curl_req
Pull request #19 (2024-12-04 21:31:50)

Auth and run steps
Supersedes #14 

This builds upon #14 to add more auto options and at the same time some finetuning options which req steps are run before generating the curl command

kevinschweikert: i will add some more thest to verify all the auth options
kevinschweikert: Hey @derekkraan, this should be ready now! I've added netrc/more auth handling to generation and parsing and refined the functionality to define which steps to run but i'm still not sure if we should implement that. What is your opinion? The only current benefit would be, that req would not try to read in the netrc file and raise if not found. Then you would need to skip the `:auth` step
derekkraan: @kevinschweikert Is the `:only` and `:except` options needed to support a concrete use-case? Is your scenario that the `--netrc` option is given, but that you can override this with `except: [:auth]`?

In that case, I think it's fine to add it. If I am misunderstanding, then please let me know.

Can you add @acalejos as a co-author to the relevant commits that he helped to author? GitHub has [a guide](https://docs.github.com/en/pull-requests/committing-changes-to-your-project/creating-and-editing-commits/creating-a-commit-with-multiple-authors) on how to do that.
derekkraan: And we should also record this in the changelog.
kevinschweikert: @derekkraan thanks for the feedback. The use case is exactly how you described. The auth step would try to read in the netrc file and raises when it does not exists. We wouldn't need this to run, because we get the filepath from the `Req.Request` options and let cURL read in the file at execution time.

Will clean up the commits and add the co-author...must have lost it in a rebase. Thanks for the hint!

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4331ede..33c7081 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,8 @@
 ## 0.98.7
 
 - Add new supported flags: `--proxy` and `--proxy-user`
+- Add more supported auth steps: `netrc` and `netrc_file`
+- Add option to exclude `req` steps to run when generating the cURL command
 
 ## 0.98.6
 - Handle `--data-raw` and `--data-ascii` ([#16](https://github.com/derekkraan/curl_req/pull/16))
diff --git a/lib/curl_req.ex b/lib/curl_req.ex
index 6bf83be..077a318 100644
--- a/lib/curl_req.ex
+++ b/lib/curl_req.ex
@@ -34,8 +34,30 @@ defmodule CurlReq do
     req
   end
 
-  defp run_steps(req) do
-    Enum.reduce(req.request_steps, req, fn {step_name, step}, req ->
+  @spec step_names(Req.Request.t(), boolean()) :: [atom()]
+  defp step_names(%Req.Request{} = _req, false), do: []
+  defp step_names(%Req.Request{} = req, true), do: req.request_steps |> Keyword.keys()
+
+  @spec step_names(Req.Request.t(), [atom()]) :: [atom()]
+  defp step_names(%Req.Request{} = req, except: excludes) do
+    for {name, _} <- req.request_steps, name not in excludes do
+      name
+    end
+  end
+
+  defp step_names(%Req.Request{} = req, only: includes) do
+    for {name, _} <- req.request_steps, name in includes do
+      name
+    end
+  end
+
+  @spec run_steps(Req.Request.t(), [atom()]) :: Req.Request.t()
+  defp run_steps(req, steps) do
+    req.request_steps
+    |> Enum.filter(fn {step, _} ->
+      step in steps
+    end)
+    |> Enum.reduce(req, fn {step_name, step}, req ->
       case step.(req) do
         {_req, _response_or_error} ->
           raise "The request was stopped by #{step_name} request_step."
@@ -60,10 +82,17 @@ defmodule CurlReq do
   * `--data-raw`
   * `-x`/`--proxy`
   * `-U`/`--proxy-user`
+  * `-u`/`--user`
+  * `-n`/`--netrc`
+  * `--netrc-file`
 
   Options:
 
-  - `run_steps`: Run the Req.Steps before generating the curl command. Default: `true`. This option is semi-private, introduced to support CurlReq.Plugin.
+  - `run_steps`: Run the Req.Steps before generating the curl command to have fine-tuned control over the Req.Request. Default: `true`. 
+    * `true`: Run all steps
+    * `false`: Run no steps
+    * `only: [atom()]`: A list of step names as atoms and only they will be executed
+    * `except: [atom()]`: A list of step names as atoms and these steps will be excluded from the executed steps
   - `flags`: Specify the style the argument flags are constructed. Can either be `:short` or `:long`, Default: `:short`
   - `flavor` or `flavour`: With the `:curl` flavor (the default) it will try to use native curl representations for compression, auth and will use the native user agent. 
   If flavor is set to `:req` the headers will not be modified and the curl command is constructed to stay as true as possible to the original `Req.Request`
@@ -78,6 +107,9 @@ defmodule CurlReq do
       ...> |> CurlReq.to_curl(flags: :long, flavor: :req)
       ~S(curl --header "accept-encoding: gzip" --header "user-agent: req/#{@req_version}" --request GET https://www.example.com)
 
+      iex> Req.new(url: "https://www.example.com")
+      ...> |> CurlReq.to_curl(run_steps: [except: [:compressed]])
+      ~S(curl -X GET https://www.example.com)
   """
   @type flags :: :short | :long
   @type flavor :: :curl | :req
@@ -85,20 +117,17 @@ defmodule CurlReq do
           flags: flags(),
           flavor: flavor(),
           flavour: flavor(),
-          run_steps: boolean()
+          run_steps: boolean() | [only: [atom()]] | [except: [atom()]]
         ]
   @spec to_curl(Req.Request.t(), to_curl_opts()) :: String.t()
   def to_curl(req, options \\ []) do
-    flag_style = Keyword.get(options, :flags, :short)
-    flavor = Keyword.get(options, :flavor, nil) || Keyword.get(options, :flavour, :curl)
-    run_steps? = Keyword.get(options, :run_steps, true)
+    opts = Keyword.validate!(options, flags: :short, run_steps: true, flavor: nil, flavour: :curl)
+    flavor = opts[:flavor] || opts[:flavour]
+    flag_style = opts[:flags]
+    run_steps = opts[:run_steps]
 
-    req =
-      if run_steps? do
-        run_steps(req)
-      else
-        req
-      end
+    available_steps = step_names(req, run_steps)
+    req = run_steps(req, available_steps)
 
     cookies =
       case Map.get(req.headers, "cookie") do
@@ -124,10 +153,7 @@ defmodule CurlReq do
 
         # avoids duplicate compression argument
         %{compressed: true} ->
-          if run_steps?, do: [], else: [compressed_flag()]
-
-        %{auth: {:basic, credentials}} ->
-          [user_flag(flag_style), credentials] ++ [basic_auth_flag()]
+          if :compressed in available_steps, do: [], else: [compressed_flag()]
 
         %{connect_options: connect_options} ->
           proxy =
@@ -151,6 +177,29 @@ defmodule CurlReq do
           []
       end
 
+    auth =
+      with %{auth: scheme} <- req.options do
+        case scheme do
+          {:bearer, token} ->
+            [header_flag(flag_style), "authorization: Bearer #{token}"]
+
+          {:basic, userinfo} ->
+            [user_flag(flag_style), userinfo] ++ [basic_auth_flag()]
+
+          :netrc ->
+            [netrc_flag(flag_style)]
+
+          {:netrc, filepath} ->
+            [netrc_file_flag(flag_style), filepath]
+
+          _ ->
+            []
+        end
+      else
+        _ ->
+          []
+      end
+
     method =
       case req.method do
         nil -> [request_flag(flag_style), "GET"]
@@ -162,7 +211,7 @@ defmodule CurlReq do
 
     CurlReq.Shell.cmd_to_string(
       "curl",
-      headers ++ cookies ++ body ++ options ++ method ++ url
+      auth ++ headers ++ cookies ++ body ++ options ++ method ++ url
     )
   end
 
@@ -173,8 +222,8 @@ defmodule CurlReq do
     [compressed_flag()]
   end
 
-  # filter out basic auth header because we expect it to be set as an auth step option
-  defp map_header({"authorization", ["Basic " <> _credentials]}, _flag_style, :curl),
+  # filter out auth header because we expect it to be set as an auth step option
+  defp map_header({"authorization", _}, _flag_style, :curl),
     do: []
 
   # filter out user agent when mode is :curl
@@ -214,6 +263,11 @@ defmodule CurlReq do
   defp proxy_user_flag(:short), do: "-U"
   defp proxy_user_flag(:long), do: "--proxy-user"
 
+  defp netrc_flag(:short), do: "-n"
+  defp netrc_flag(:long), do: "--netrc"
+
+  defp netrc_file_flag(_), do: "--netrc-file"
+
   @doc """
   Transforms a curl command into a Req request.
 
diff --git a/lib/curl_req/macro.ex b/lib/curl_req/macro.ex
index 4bac74c..e414800 100644
--- a/lib/curl_req/macro.ex
+++ b/lib/curl_req/macro.ex
@@ -27,7 +27,9 @@ defmodule CurlReq.Macro do
           user: :string,
           compressed: :boolean,
           proxy: :string,
-          proxy_user: :string
+          proxy_user: :string,
+          netrc: :boolean,
+          netrc_file: :string
         ],
         aliases: [
           H: :header,
@@ -39,7 +41,8 @@ defmodule CurlReq.Macro do
           L: :location,
           u: :user,
           x: :proxy,
-          U: :proxy_user
+          U: :proxy_user,
+          n: :netrc
         ]
       )
 
@@ -103,6 +106,12 @@ defmodule CurlReq.Macro do
             |> Req.Request.prepend_request_steps(auth: &Req.Steps.auth/1)
             |> Req.merge(auth: {:bearer, token})
 
+          {"authorization", "Basic " <> token} ->
+            req
+            |> Req.Request.register_options([:auth])
+            |> Req.Request.prepend_request_steps(auth: &Req.Steps.auth/1)
+            |> Req.merge(auth: {:basic, token})
+
           _ ->
             Req.Request.put_header(req, key, value)
         end
@@ -168,15 +177,39 @@ defmodule CurlReq.Macro do
   end
 
   defp add_auth(req, options) do
-    case Keyword.get(options, :user) do
+    req =
+      case Keyword.get(options, :user) do
+        nil ->
+          req
+
+        credentials ->
+          req
+          |> Req.Request.register_options([:auth])
+          |> Req.Request.prepend_request_steps(auth: &Req.Steps.auth/1)
+          |> Req.merge(auth: {:basic, credentials})
+      end
+
+    req =
+      case Keyword.get(options, :netrc) do
+        nil ->
+          req
+
+        true ->
+          req
+          |> Req.Request.register_options([:auth])
+          |> Req.Request.prepend_request_steps(auth: &Req.Steps.auth/1)
+          |> Req.merge(auth: :netrc)
+      end
+
+    case Keyword.get(options, :netrc_file) do
       nil ->
         req
 
-      credentials ->
+      path ->
         req
         |> Req.Request.register_options([:auth])
         |> Req.Request.prepend_request_steps(auth: &Req.Steps.auth/1)
-        |> Req.merge(auth: {:basic, credentials})
+        |> Req.merge(auth: {:netrc, path})
     end
   end
 


Test patch:
diff --git a/test/curl_req/macro_test.exs b/test/curl_req/macro_test.exs
index cf91ba7..59b5d35 100644
--- a/test/curl_req/macro_test.exs
+++ b/test/curl_req/macro_test.exs
@@ -134,7 +134,7 @@ defmodule CurlReq.MacroTest do
                }
     end
 
-    test "auth" do
+    test "basic auth" do
       assert ~CURL(curl http://example.com -u user:pass) ==
                %Req.Request{
                  url: URI.parse("http://example.com"),
@@ -171,6 +171,30 @@ defmodule CurlReq.MacroTest do
                }
     end
 
+    test "netrc auth" do
+      assert ~CURL(curl http://example.com -n) ==
+               %Req.Request{
+                 url: URI.parse("http://example.com"),
+                 body: nil,
+                 registered_options: MapSet.new([:auth]),
+                 options: %{auth: :netrc},
+                 current_request_steps: [:auth],
+                 request_steps: [auth: &Req.Steps.auth/1]
+               }
+    end
+
+    test "netrc file auth" do
+      assert ~CURL(curl http://example.com --netrc-file "./mynetrc") ==
+               %Req.Request{
+                 url: URI.parse("http://example.com"),
+                 body: nil,
+                 registered_options: MapSet.new([:auth]),
+                 options: %{auth: {:netrc, "./mynetrc"}},
+                 current_request_steps: [:auth],
+                 request_steps: [auth: &Req.Steps.auth/1]
+               }
+    end
+
     test "compressed" do
       assert ~CURL(curl --compressed http://example.com) ==
                %Req.Request{
diff --git a/test/curl_req_test.exs b/test/curl_req_test.exs
index ce34e26..bb1ce1a 100644
--- a/test/curl_req_test.exs
+++ b/test/curl_req_test.exs
@@ -85,12 +85,6 @@ defmodule CurlReqTest do
                |> CurlReq.to_curl(flavor: :req)
     end
 
-    test "basic auth option" do
-      assert "curl --compressed -u user:pass --basic -X GET https://example.com" ==
-               Req.new(url: "https://example.com", auth: {:basic, "user:pass"})
-               |> CurlReq.to_curl()
-    end
-
     test "proxy" do
       assert ~S(curl --compressed -x "http://my.proxy.com:80" -X GET https://example.com) ==
                Req.new(
@@ -113,5 +107,64 @@ defmodule CurlReqTest do
                )
                |> CurlReq.to_curl()
     end
+
+    test "basic auth option" do
+      assert "curl -u user:pass --basic --compressed -X GET https://example.com" ==
+               Req.new(url: "https://example.com", auth: {:basic, "user:pass"})
+               |> CurlReq.to_curl()
+    end
+
+    test "bearer auth option" do
+      assert ~S(curl -H "authorization: Bearer foo123bar" --compressed -X GET https://example.com) ==
+               Req.new(url: "https://example.com", auth: {:bearer, "foo123bar"})
+               |> CurlReq.to_curl()
+    end
+
+    @tag :tmp_dir
+    test "netrc auth option", %{tmp_dir: tmp_dir} do
+      credentials =
+        """
+        machine example.com
+          login foo
+          password bar
+        """
+
+      netrc_path = Path.join(tmp_dir, "my_netrc")
+      File.write(netrc_path, credentials)
+      System.put_env("NETRC", netrc_path)
+
+      assert "curl -n --compressed -X GET https://example.com" ==
+               Req.new(url: "https://example.com", auth: :netrc)
+               |> CurlReq.to_curl()
+    end
+
+    @tag :tmp_dir
+    test "netrc file auth option", %{tmp_dir: tmp_dir} do
+      credentials =
+        """
+        machine example.com
+          login foo
+          password bar
+        """
+
+      netrc_path = Path.join(tmp_dir, "my_netrc")
+      File.write(netrc_path, credentials)
+
+      assert ~s(curl --netrc-file "#{netrc_path}" --compressed -X GET https://example.com) ==
+               Req.new(url: "https://example.com", auth: {:netrc, netrc_path})
+               |> CurlReq.to_curl()
+    end
+
+    test "include `encode_body` does not run `comporessed` or other steps" do
+      assert ~S(curl -H "accept: application/json" -H "content-type: application/json" -d "{\"key\":\"val\"}" -X GET https://example.com) ==
+               Req.new(url: "https://example.com", json: %{key: "val"})
+               |> CurlReq.to_curl(run_steps: [only: [:encode_body]])
+    end
+
+    test "exclude `compressed` and `encode_body` do not run" do
+      assert "curl -X GET https://example.com" ==
+               Req.new(url: "https://example.com", json: %{key: "val"})
+               |> CurlReq.to_curl(run_steps: [except: [:compressed, :encode_body]])
+    end
   end
 end