Repository: derekkraan/curl_req
Pull request #5 (2024-06-05 10:06:35)

Add --head, --form, --user and --location flags to parser
I've added some more flags to parse from the curl command. This should cover most use cases. Only missing thing could be to support the `@` syntax in, for example, `curl [URL] -d @filename.txt` to load from a file. We could read the file in Elixir, but I don't know if we should support it.

I didn't yet do the other direction in `to_curl` in this PR for the new flags. Should i add them here or could this be a seperate PR?

derekkraan: I would be happy with a note in the documentation that we don't implement the read file syntax. If someone needs it, they can build it and contribute back a PR.

Re: to_curl implementations of these options. You can either add in this PR or add a TODO at the bottom of the README to cover it.

If you make those doc changes then I can merge this one.

Thanks for all your contributions btw!
derekkraan: Could you also add an entry to the CHANGELOG.md?
kevinschweikert: Thanks! I will update the PR tonight
kevinschweikert: Hey @derekkraan i've extended the PR with your suggestions and took some time to clean up the documentation and test files. Hope that's okay for you!

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4d97d62..ad543d5 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,9 @@
 
 ## 0.98.4
 - Add CurlReq.Plugin
+- Add new supported flags: `--head`, `--form`, `--user` and `--location`
+- Add `CurlReq.from_curl/1`
+- Improved docs and added typespecs
 
 ## 0.98.3
 - Change `ex_doc` to a dev dependency.
diff --git a/README.md b/README.md
index fb0d180..ad7514d 100644
--- a/README.md
+++ b/README.md
@@ -54,6 +54,8 @@ iex> Req.new(url: "/fact", base_url: "https://catfact.ninja/")
 
 ```
 
+<!-- MDOC !-->
+
 ## Installation
 
 The package can be installed
diff --git a/lib/curl_req.ex b/lib/curl_req.ex
index 965cb6e..f307f7a 100644
--- a/lib/curl_req.ex
+++ b/lib/curl_req.ex
@@ -6,7 +6,6 @@ defmodule CurlReq do
              |> Enum.fetch!(1)
 
   @type inspect_opt :: {:label, String.t()}
-  @type req_request :: %Req.Request{}
 
   @doc """
   Inspect a Req struct in curl syntax.
@@ -19,7 +18,7 @@ defmodule CurlReq do
       ...> # |> Req.request!()
 
   """
-  @spec inspect(req_request(), [inspect_opt()]) :: req_request()
+  @spec inspect(Req.Request.t(), [inspect_opt()]) :: Req.Request.t()
   def inspect(req, opts \\ []) do
     case Keyword.get(opts, :label) do
       nil -> IO.puts(to_curl(req))
@@ -44,6 +43,14 @@ defmodule CurlReq do
   @doc """
   Transforms a Req request into a curl command.
 
+  Supported curl flags are:
+
+  * `-b`
+  * `-H`
+  * `-X`
+  * `-I`
+  * `-d`
+
   ## Examples
 
       iex> Req.new(url: URI.parse("https://www.google.com"))
@@ -51,7 +58,7 @@ defmodule CurlReq do
       ~S(curl -H "accept-encoding: gzip" -H "user-agent: req/0.4.14" -X GET https://www.google.com)
 
   """
-  @spec to_curl(req_request()) :: String.t()
+  @spec to_curl(Req.Request.t(), Keyword.t()) :: String.t()
   def to_curl(req, options \\ []) do
     req =
       if Keyword.get(options, :run_steps, true) do
@@ -79,30 +86,92 @@ defmodule CurlReq do
         body -> ["-d", body]
       end
 
+    redirect =
+      case req.options do
+        %{redirect: true} -> ["-L"]
+        _ -> []
+      end
+
     method =
       case req.method do
         nil -> ["-X", "GET"]
+        :head -> ["-I"]
         m -> ["-X", String.upcase(to_string(m))]
       end
 
     url = [to_string(req.url)]
 
-    CurlReq.Shell.cmd_to_string("curl", headers ++ cookies ++ body ++ method ++ url)
+    CurlReq.Shell.cmd_to_string(
+      "curl",
+      headers ++ cookies ++ body ++ method ++ redirect ++ url
+    )
   end
 
   @doc """
   Transforms a curl command into a Req request.
 
+  Supported curl command line flags are supported:
+
+  * `-H`/`--header`
+  * `-X`/`--request`
+  * `-d`/`--data`
+  * `-b`/`--cookie`
+  * `-I`/`--head`
+  * `-F`/`--form`
+  * `-L`/`--location`
+  * `-u`/`--user`
+
+  The `curl` command prefix is optional
+
+  > #### Info {: .info}
+  >
+  > Only string inputs are supported. That means for example `-d @data.txt` will not load the file or `-d @-` will not read from stdin
+
+  ## Examples
+
+      iex> CurlReq.from_curl("curl https://www.google.com")
+      %Req.Request{method: :get, url: URI.parse("https://www.google.com")}
+
+      iex> ~S(curl -d "some data" https://example.com) |> CurlReq.from_curl()
+      %Req.Request{method: :get, body: "some data", url: URI.parse("https://example.com")}
+
+      iex> CurlReq.from_curl("curl -I https://example.com")
+      %Req.Request{method: :head, url: URI.parse("https://example.com")}
+
+      iex> CurlReq.from_curl("curl -b cookie_key=cookie_val https://example.com")
+      %Req.Request{method: :get, headers: %{"cookie" => ["cookie_key=cookie_val"]}, url: URI.parse("https://example.com")}
+  """
+  @doc since: "0.98.4"
+
+  @spec from_curl(String.t()) :: Req.Request.t()
+  def from_curl(curl_command), do: CurlReq.Macro.parse(curl_command)
+
+  @doc """
+  Same as `from_curl/1` but as a sigil. The benefit here is, that the Req.Request struct will be created at compile time and you don't need to escape the string
+
   ## Examples
 
       iex> import CurlReq
       ...> ~CURL(curl "https://www.google.com")
       %Req.Request{method: :get, url: URI.parse("https://www.google.com")}
+
+      iex> import CurlReq
+      ...> ~CURL(curl -d "some data" "https://example.com")
+      %Req.Request{method: :get, body: "some data", url: URI.parse("https://example.com")}
+
+      iex> import CurlReq
+      ...> ~CURL(curl -I "https://example.com")
+      %Req.Request{method: :head, url: URI.parse("https://example.com")}
+
+      iex> import CurlReq
+      ...> ~CURL(curl -b "cookie_key=cookie_val" "https://example.com")
+      %Req.Request{method: :get, headers: %{"cookie" => ["cookie_key=cookie_val"]}, url: URI.parse("https://example.com")}
   """
+  defmacro sigil_CURL(curl_command, modifiers)
+
   defmacro sigil_CURL({:<<>>, _line_info, [command]}, _extra) do
     command
     |> CurlReq.Macro.parse()
-    |> CurlReq.Macro.to_req()
     |> Macro.escape()
   end
 end
diff --git a/lib/curl_req/macro.ex b/lib/curl_req/macro.ex
index 4425932..609c74b 100644
--- a/lib/curl_req/macro.ex
+++ b/lib/curl_req/macro.ex
@@ -3,6 +3,7 @@ defmodule CurlReq.Macro do
 
   # TODO: handle newlines
 
+  @spec parse(String.t()) :: Req.Request.t()
   def parse(command) do
     command =
       command
@@ -13,22 +14,39 @@ defmodule CurlReq.Macro do
       command
       |> OptionParser.split()
       |> OptionParser.parse(
-        strict: [header: :keep, request: :string, data: :keep, cookie: :string],
-        aliases: [H: :header, X: :request, d: :data, b: :cookie]
+        strict: [
+          header: :keep,
+          request: :string,
+          data: :keep,
+          cookie: :string,
+          head: :boolean,
+          form: :keep,
+          location: :boolean,
+          user: :string
+        ],
+        aliases: [
+          H: :header,
+          X: :request,
+          d: :data,
+          b: :cookie,
+          I: :head,
+          F: :form,
+          L: :location,
+          u: :user
+        ]
       )
 
     url = String.trim(url)
-    %{url: url, options: options}
-  end
 
-  @doc false
-  def to_req(%{url: url, options: options}) do
     %Req.Request{}
     |> Req.merge(url: url)
     |> add_header(options)
     |> add_method(options)
     |> add_body(options)
     |> add_cookie(options)
+    |> add_form(options)
+    |> add_auth(options)
+    |> configure_redirects(options)
   end
 
   defp add_header(req, options) do
@@ -46,10 +64,14 @@ defmodule CurlReq.Macro do
 
   defp add_method(req, options) do
     method =
-      options
-      |> Keyword.get(:request, "GET")
-      |> String.downcase()
-      |> String.to_existing_atom()
+      if Keyword.get(options, :head, false) do
+        :head
+      else
+        options
+        |> Keyword.get(:request, "GET")
+        |> String.downcase()
+        |> String.to_existing_atom()
+      end
 
     Req.merge(req, method: method)
   end
@@ -70,4 +92,48 @@ defmodule CurlReq.Macro do
       cookie -> Req.Request.put_header(req, "cookie", cookie)
     end
   end
+
+  defp add_form(req, options) do
+    case Keyword.get_values(options, :form) do
+      [] ->
+        req
+
+      formdata ->
+        form =
+          for fd <- formdata, reduce: %{} do
+            map ->
+              [key, value] = String.split(fd, "=", parts: 2)
+              Map.put(map, key, value)
+          end
+
+        req
+        |> Req.Request.register_options([:form])
+        |> Req.Request.prepend_request_steps(encode_body: &Req.Steps.encode_body/1)
+        |> Req.merge(form: form)
+    end
+  end
+
+  defp add_auth(req, options) do
+    case Keyword.get(options, :user) do
+      nil ->
+        req
+
+      credentials ->
+        req
+        |> Req.Request.register_options([:auth])
+        |> Req.Request.prepend_request_steps(auth: &Req.Steps.auth/1)
+        |> Req.merge(auth: {:basic, credentials})
+    end
+  end
+
+  defp configure_redirects(req, options) do
+    if Keyword.get(options, :location, false) do
+      req
+      |> Req.Request.register_options([:redirect])
+      |> Req.Request.prepend_response_steps(redirect: &Req.Steps.redirect/1)
+      |> Req.merge(redirect: true)
+    else
+      req
+    end
+  end
 end


Test patch:
diff --git a/test/curl_req/macro_test.exs b/test/curl_req/macro_test.exs
new file mode 100644
index 0000000..fe0862c
--- /dev/null
+++ b/test/curl_req/macro_test.exs
@@ -0,0 +1,125 @@
+defmodule CurlReq.MacroTest do
+  use ExUnit.Case, async: true
+
+  import CurlReq
+
+  describe "macro" do
+    test "single header" do
+      assert ~CURL(curl -H "user-agent: req/0.4.14" -X GET https://catfact.ninja/fact) ==
+               %Req.Request{
+                 method: :get,
+                 headers: %{"user-agent" => ["req/0.4.14"]},
+                 url: URI.parse("https://catfact.ninja/fact")
+               }
+    end
+
+    test "post method" do
+      assert ~CURL(curl -X POST https://example.com) ==
+               %Req.Request{
+                 method: :post,
+                 url: URI.parse("https://example.com")
+               }
+    end
+
+    test "head method" do
+      assert ~CURL(curl -I https://example.com) ==
+               %Req.Request{
+                 method: :head,
+                 url: URI.parse("https://example.com")
+               }
+    end
+
+    test "multiple headers with body" do
+      assert ~CURL(curl -H "accept-encoding: gzip" -H "authorization: Bearer 6e8f18e6-141b-4d12-8397-7e7791d92ed4:lon" -H "content-type: application/json" -H "user-agent: req/0.4.14" -d "{\"input\":[{\"leadFormFields\":{\"Company\":\"k\",\"Country\":\"DZ\",\"Email\":\"k\",\"FirstName\":\"k\",\"Industry\":\"CTO\",\"LastName\":\"k\",\"Phone\":\"k\",\"PostalCode\":\"1234ZZ\",\"jobspecialty\":\"engineer\",\"message\":\"I would like to know if Roche delivers to The Netherlands.\"}}],\"formId\":4318}" -X POST "https://example.com/rest/v1/leads/submitForm.json") ==
+               %Req.Request{
+                 method: :post,
+                 url: URI.parse("https://example.com/rest/v1/leads/submitForm.json"),
+                 headers: %{
+                   "accept-encoding" => ["gzip"],
+                   "authorization" => ["Bearer 6e8f18e6-141b-4d12-8397-7e7791d92ed4:lon"],
+                   "content-type" => ["application/json"],
+                   "user-agent" => ["req/0.4.14"]
+                 },
+                 body:
+                   "{\"input\":[{\"leadFormFields\":{\"Company\":\"k\",\"Country\":\"DZ\",\"Email\":\"k\",\"FirstName\":\"k\",\"Industry\":\"CTO\",\"LastName\":\"k\",\"Phone\":\"k\",\"PostalCode\":\"1234ZZ\",\"jobspecialty\":\"engineer\",\"message\":\"I would like to know if Roche delivers to The Netherlands.\"}}],\"formId\":4318}"
+               }
+    end
+
+    test "without curl prefix" do
+      assert ~CURL(http://localhost) ==
+               %Req.Request{
+                 method: :get,
+                 url: URI.parse("http://localhost")
+               }
+    end
+
+    test "multiple data flags" do
+      assert ~CURL(curl http://example.com -d name=foo -d mail=bar) ==
+               %Req.Request{
+                 url: URI.parse("http://example.com"),
+                 body: "name=foo&mail=bar"
+               }
+    end
+
+    test "cookie" do
+      assert ~CURL(http://example.com -b "name1=value1") ==
+               %Req.Request{
+                 url: URI.parse("http://example.com"),
+                 headers: %{"cookie" => ["name1=value1"]}
+               }
+
+      assert ~CURL(http://example.com -b "name1=value1; name2=value2") ==
+               %Req.Request{
+                 url: URI.parse("http://example.com"),
+                 headers: %{"cookie" => ["name1=value1; name2=value2"]}
+               }
+    end
+
+    test "formdata" do
+      assert ~CURL(curl http://example.com -F name=foo -F mail=bar) ==
+               %Req.Request{
+                 url: URI.parse("http://example.com"),
+                 body: nil,
+                 registered_options: MapSet.new([:form]),
+                 options: %{form: %{"name" => "foo", "mail" => "bar"}},
+                 current_request_steps: [:encode_body],
+                 request_steps: [encode_body: &Req.Steps.encode_body/1]
+               }
+    end
+
+    test "auth" do
+      assert ~CURL(curl http://example.com -u user:pass) ==
+               %Req.Request{
+                 url: URI.parse("http://example.com"),
+                 body: nil,
+                 registered_options: MapSet.new([:auth]),
+                 options: %{auth: {:basic, "user:pass"}},
+                 current_request_steps: [:auth],
+                 request_steps: [auth: &Req.Steps.auth/1]
+               }
+    end
+
+    test "redirect" do
+      assert ~CURL(curl -L http://example.com) ==
+               %Req.Request{
+                 url: URI.parse("http://example.com"),
+                 registered_options: MapSet.new([:redirect]),
+                 options: %{redirect: true},
+                 response_steps: [redirect: &Req.Steps.redirect/1]
+               }
+    end
+
+    test "cookie, formadata, auth and redirect" do
+      assert ~CURL(curl -L -u user:pass -F name=foo -b name=bar http://example.com) ==
+               %Req.Request{
+                 url: URI.parse("http://example.com"),
+                 headers: %{"cookie" => ["name=bar"]},
+                 current_request_steps: [:auth, :encode_body],
+                 registered_options: MapSet.new([:redirect, :auth, :form]),
+                 options: %{redirect: true, auth: {:basic, "user:pass"}, form: %{"name" => "foo"}},
+                 request_steps: [auth: &Req.Steps.auth/1, encode_body: &Req.Steps.encode_body/1],
+                 response_steps: [redirect: &Req.Steps.redirect/1]
+               }
+    end
+  end
+end
diff --git a/test/curl_req_test.exs b/test/curl_req_test.exs
index 1b84de5..55a1dbd 100644
--- a/test/curl_req_test.exs
+++ b/test/curl_req_test.exs
@@ -20,6 +20,24 @@ defmodule CurlReqTest do
                "curl #{default_header()} -b \"name1=value1\" -X GET http://example.com"
     end
 
+    test "redirect flag gets set" do
+      assert Req.new(url: "http://example.com", redirect: true)
+             |> CurlReq.to_curl() ==
+               "curl #{default_header()} -X GET -L http://example.com"
+    end
+
+    test "head method flag gets set" do
+      assert Req.new(url: "http://example.com", method: :head)
+             |> CurlReq.to_curl() ==
+               "curl #{default_header()} -I http://example.com"
+    end
+
+    test "formdata flags get set with correct headers and body" do
+      assert Req.new(url: "http://example.com", form: [key1: "value1", key2: "value2"])
+             |> CurlReq.to_curl() ==
+               "curl -H \"accept-encoding: gzip\" -H \"content-type: application/x-www-form-urlencoded\" -H \"user-agent: req/#{@req_version}\" -d \"key1=value1&key2=value2\" -X GET http://example.com"
+    end
+
     test "works when body is iodata" do
       assert "curl #{default_header()} -d hello -X POST https://catfact.ninja/fact" ==
                Req.new(
@@ -31,61 +49,4 @@ defmodule CurlReqTest do
                |> CurlReq.to_curl()
     end
   end
-
-  describe "macro" do
-    test "single header" do
-      assert ~CURL(curl -H "user-agent: req/0.4.14" -X GET https://catfact.ninja/fact) ==
-               %Req.Request{
-                 method: :get,
-                 headers: %{"user-agent" => ["req/0.4.14"]},
-                 url: URI.parse("https://catfact.ninja/fact")
-               }
-    end
-
-    test "multiple headers with body" do
-      assert ~CURL(curl -H "accept-encoding: gzip" -H "authorization: Bearer 6e8f18e6-141b-4d12-8397-7e7791d92ed4:lon" -H "content-type: application/json" -H "user-agent: req/0.4.14" -d "{\"input\":[{\"leadFormFields\":{\"Company\":\"k\",\"Country\":\"DZ\",\"Email\":\"k\",\"FirstName\":\"k\",\"Industry\":\"CTO\",\"LastName\":\"k\",\"Phone\":\"k\",\"PostalCode\":\"1234ZZ\",\"jobspecialty\":\"engineer\",\"message\":\"I would like to know if Roche delivers to The Netherlands.\"}}],\"formId\":4318}" -X POST "https://example.com/rest/v1/leads/submitForm.json") ==
-               %Req.Request{
-                 method: :post,
-                 url: URI.parse("https://example.com/rest/v1/leads/submitForm.json"),
-                 headers: %{
-                   "accept-encoding" => ["gzip"],
-                   "authorization" => ["Bearer 6e8f18e6-141b-4d12-8397-7e7791d92ed4:lon"],
-                   "content-type" => ["application/json"],
-                   "user-agent" => ["req/0.4.14"]
-                 },
-                 body:
-                   "{\"input\":[{\"leadFormFields\":{\"Company\":\"k\",\"Country\":\"DZ\",\"Email\":\"k\",\"FirstName\":\"k\",\"Industry\":\"CTO\",\"LastName\":\"k\",\"Phone\":\"k\",\"PostalCode\":\"1234ZZ\",\"jobspecialty\":\"engineer\",\"message\":\"I would like to know if Roche delivers to The Netherlands.\"}}],\"formId\":4318}"
-               }
-    end
-
-    test "without curl prefix" do
-      assert ~CURL(http://localhost) ==
-               %Req.Request{
-                 method: :get,
-                 url: URI.parse("http://localhost")
-               }
-    end
-
-    test "multiple data flags" do
-      assert ~CURL(curl http://example.com -d name=foo -d mail=bar) ==
-               %Req.Request{
-                 url: URI.parse("http://example.com"),
-                 body: "name=foo&mail=bar"
-               }
-    end
-
-    test "cookie" do
-      assert ~CURL(http://example.com -b "name1=value1") ==
-               %Req.Request{
-                 url: URI.parse("http://example.com"),
-                 headers: %{"cookie" => ["name1=value1"]}
-               }
-
-      assert ~CURL(http://example.com -b "name1=value1; name2=value2") ==
-               %Req.Request{
-                 url: URI.parse("http://example.com"),
-                 headers: %{"cookie" => ["name1=value1; name2=value2"]}
-               }
-    end
-  end
 end