Repository: elastic/logstash
Pull request #17316: [8.16] Surface failures from nested rake/shell tasks (backport #17310)
Author: login: mergify[bot]
Labels: name: backport
name: conflicts
description: Detected git conflicts
Created: 2025-03-13T22:22:19Z | Merged: 2025-03-20T15:34:31Z

## Release notes

[rn:skip]

## What does this PR do?



Previously when rake would shell out the output would be lost. This made debugging CI logs difficult. This commit updates the stack with improved message surfacing on error.

Additionally this commit backports a change https://github.com/elastic/logstash/pull/16660 to allow linting specific files as it was originally missed and modifies the same method signature as we update in this backport. 
'
## Why is it important/What is the impact to the user?
N/A


## Checklist



- [x] My code follows the style guidelines of this project
- [ ] I have commented my code, particularly in hard-to-understand areas
- ~[ ] I have made corresponding changes to the documentation~
- ~[ ] I have made corresponding change to the default configuration files (and/or docker env variables)~
- ~[ ] I have added tests that prove my fix is effective or that my feature works~

## Related issues


- https://github.com/elastic/logstash/pull/17298
- https://github.com/elastic/logstash/pull/16660

## Logs


<hr>This is an automatic backport of pull request #17310 done by [Mergify](https://mergify.com).


Diff:
diff --git a/build.gradle b/build.gradle
index 3d32b1dc102..14b3754f994 100644
--- a/build.gradle
+++ b/build.gradle
@@ -763,10 +763,16 @@ class JDKDetails {
 }
 
 tasks.register("lint") {
-    // Calls rake's 'lint' task
+    description = "Lint Ruby source files. Use -PrubySource=file1.rb,file2.rb to specify files"
     dependsOn installDevelopmentGems
     doLast {
-        rake(projectDir, buildDir, 'lint:report')
+        if (project.hasProperty("rubySource")) {
+            // Split the comma-separated files and pass them as separate arguments
+            def files = project.property("rubySource").split(",")
+            rake(projectDir, buildDir, "lint:report", *files)
+        } else {
+            rake(projectDir, buildDir, "lint:report")
+        }
     }
 }
 
diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake
index 0fcf0570d36..5547d531131 100644
--- a/rakelib/artifacts.rake
+++ b/rakelib/artifacts.rake
@@ -15,6 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+require 'shellwords'
+
 namespace "artifact" do
   SNAPSHOT_BUILD = ENV["RELEASE"] != "1"
   VERSION_QUALIFIER = ENV["VERSION_QUALIFIER"].to_s.strip.empty? ? nil : ENV["VERSION_QUALIFIER"]
@@ -127,12 +129,34 @@ namespace "artifact" do
     result
   end
 
-  # execute Kernel#system call,checking the exist status of the executed command and eventually reporting as exception
+  ##
+  # @override safe_system([env,] command... [,options])
+  # execute Kernel#system call,checking the exit status of the executed command and eventually reporting as exception
   def safe_system(*args)
-    if !system(*args)
-      status = $?
+    command = args.dup # avoid mutating input for reporting
+    env = command.size > 1 && command.first.kind_of?(Hash) ? command.shift : {}
+    options = command.size > 1 && command.last.kind_of?(Hash) ? command.pop : {}
+    fail("unsupported options #{options}") unless options.empty?
+
+    # Normalize command to a single string from either a multi-word string
+    # or an array of individual words
+    command = command.size > 1 ? Shellwords.join(command.map(&:to_s)) : command.first.to_s
+
+    # prepend the environment
+    env.each do |k,v|
+      command.prepend("#{Shellwords.escape(k.to_s)}=#{Shellwords.escape(v.to_s)} ")
+    end
+
+    output = `#{command} 2>&1`
+    status = $?
+
+    if !status.success?
+      puts "Command failed: #{args.inspect}"
+      puts "Output: #{output}"
       raise "Got exit status #{status.exitstatus} attempting to execute #{args.inspect}!"
     end
+
+    true
   end
 
   desc "Generate rpm, deb, tar and zip artifacts"
diff --git a/rakelib/lint.rake b/rakelib/lint.rake
index c3aa4df45f0..f8d5eea0845 100644
--- a/rakelib/lint.rake
+++ b/rakelib/lint.rake
@@ -14,7 +14,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
 namespace "lint" do
   module RuboCLI
     def self.run!(*args)
@@ -25,28 +24,44 @@ namespace "lint" do
     end
   end
 
-  # task that runs lint report
-  desc "Report all Lint Cops"
-  task "report" do
-    RuboCLI.run!("--lint")
+  desc "Report all Lint Cops. Optional: Specify one or more files"
+  task :report, [:file] do |t, args|
+    files = [args[:file], *args.extras].compact
+
+    if files.empty?
+      RuboCLI.run!("--lint")
+    else
+      puts "Running lint report on specific files: #{files.join(', ')}"
+      RuboCLI.run!("--lint", *files)
+    end
   end
 
-  # Tasks automatically fixes a Cop passed as a parameter (e.g. Lint/DeprecatedClassMethods)
-  # TODO: Add a way to autocorrect all cops, and not just the one passed as parameter
-  desc "Automatically fix all instances of a Cop passed as a parameter"
-  task "correct", [:cop] do |t, args|
+  # Tasks automatically fixes a Cop passed as a parameter
+  desc "Automatically fix all instances of a Cop passed as a parameter. Optional: Specify one or more files"
+  task :correct, [:cop] do |t, args|
     if args[:cop].to_s.empty?
       puts "No Cop has been provided, aborting..."
       exit(0)
     else
-      puts "Attempting to correct Lint issues for: #{args[:cop].to_s}"
-      RuboCLI.run!("--autocorrect-all", "--only", args[:cop].to_s)
+      files = args.extras
+      if files.empty?
+        puts "Attempting to correct Lint issues for: #{args[:cop]}"
+        RuboCLI.run!("--autocorrect-all", "--only", args[:cop])
+      else
+        puts "Attempting to correct Lint issues for #{args[:cop]} in files: #{files.join(', ')}"
+        RuboCLI.run!("--autocorrect-all", "--only", args[:cop], *files)
+      end
     end
   end
 
-  # task that automatically fixes code formatting
-  desc "Automatically fix Layout Cops"
-  task "format" do
-    RuboCLI.run!("--fix-layout")
+  desc "Automatically fix Layout Cops. Optional: Specify one or more files"
+  task :format, [:file] do |t, args|
+    files = [args[:file], *args.extras].compact
+    if files.empty?
+      RuboCLI.run!("--fix-layout")
+    else
+      puts "Running format fixes on specific files: #{files.join(', ')}"
+      RuboCLI.run!("--fix-layout", *files)
+    end
   end
-end
+end
\ No newline at end of file
diff --git a/rubyUtils.gradle b/rubyUtils.gradle
index 65458c4f8e9..e443b48d9c7 100644
--- a/rubyUtils.gradle
+++ b/rubyUtils.gradle
@@ -151,17 +151,24 @@ void buildGem(File projectDir, File buildDir, String gemspec) {
  * @param projectDir Gradle projectDir
  * @param buildDir Gradle buildDir
  * @param plugin Plugin to run specs for
- * @param args CLI arguments to pass to rspec
+ * @param args Optional arguments to pass to the rake task
  */
-void rake(File projectDir, File buildDir, String task) {
+void rake(File projectDir, File buildDir, String task, String... args) {
     executeJruby projectDir, buildDir, { ScriptingContainer jruby ->
         jruby.currentDirectory = projectDir
         jruby.runScriptlet("require 'rake'; require 'time'")
+        def rakeArgs = args ? "'${args.join("','")}'" : ""
         jruby.runScriptlet("""
-                rake = Rake.application
-                rake.init
-                rake.load_rakefile
-                rake['${task}'].invoke
+                begin
+                    rake = Rake.application
+                    rake.init
+                    rake.load_rakefile
+                    rake['${task}'].invoke(${rakeArgs})
+                rescue => e
+                    puts "Rake task error: #{e.class}: #{e.message}"
+                    puts "Backtrace: #{e.backtrace.join("\\n")}"
+                    raise e
+                end
                 """
         )
     }


Reviews:
user: login: andsel
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-14T13:26:37Z
user: login: andsel
body: LGTM
state: APPROVED
author_association: MEMBER
submitted_at: 2025-03-14T13:26:47Z
user: login: donoghuc
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-18T16:46:12Z

Review comments:
diff_hunk: @@ -151,17 +151,24 @@ void buildGem(File projectDir, File buildDir, String gemspec) {
  * @param projectDir Gradle projectDir
  * @param buildDir Gradle buildDir
  * @param plugin Plugin to run specs for
- * @param args CLI arguments to pass to rspec
+ * @param args Optional arguments to pass to the rake task
  */
-void rake(File projectDir, File buildDir, String task) {
+void rake(File projectDir, File buildDir, String task, String... args) {
     executeJruby projectDir, buildDir, { ScriptingContainer jruby ->
         jruby.currentDirectory = projectDir
         jruby.runScriptlet("require 'rake'; require 'time'")
+        def rakeArgs = args ? "'${args.join("','")}'" : ""
         jruby.runScriptlet("""
-                rake = Rake.application
-                rake.init
-                rake.load_rakefile
-                rake['${task}'].invoke
+                begin
+                    rake = Rake.application
+                    rake.init
+                    rake.load_rakefile
+                    rake['${task}'].invoke(${rakeArgs})
+                rescue => e
+                    puts "Rake task error: #{e.class}: #{e.message}"
+                    puts "Backtrace: #{e.backtrace.join("\\n")}"
+                    raise e
+                end
path: rubyUtils.gradle
user: login: andsel
body: Respect to `8.17` backport here is just a missed new line https://github.com/elastic/logstash/pull/17315/files#diff-7a2156b270dee259f243cf6371cdeb28e3bfdbc8c0e418cb37cfb4724144b305R172
created_at: 2025-03-14T13:26:36Z
author_association: MEMBER
diff_hunk: @@ -151,17 +151,24 @@ void buildGem(File projectDir, File buildDir, String gemspec) {
  * @param projectDir Gradle projectDir
  * @param buildDir Gradle buildDir
  * @param plugin Plugin to run specs for
- * @param args CLI arguments to pass to rspec
+ * @param args Optional arguments to pass to the rake task
  */
-void rake(File projectDir, File buildDir, String task) {
+void rake(File projectDir, File buildDir, String task, String... args) {
     executeJruby projectDir, buildDir, { ScriptingContainer jruby ->
         jruby.currentDirectory = projectDir
         jruby.runScriptlet("require 'rake'; require 'time'")
+        def rakeArgs = args ? "'${args.join("','")}'" : ""
         jruby.runScriptlet("""
-                rake = Rake.application
-                rake.init
-                rake.load_rakefile
-                rake['${task}'].invoke
+                begin
+                    rake = Rake.application
+                    rake.init
+                    rake.load_rakefile
+                    rake['${task}'].invoke(${rakeArgs})
+                rescue => e
+                    puts "Rake task error: #{e.class}: #{e.message}"
+                    puts "Backtrace: #{e.backtrace.join("\\n")}"
+                    raise e
+                end
path: rubyUtils.gradle
user: login: donoghuc
body: Good eye. This had some merge conflicts that needed some special care. I must have slipped up on a whitespace change. 
created_at: 2025-03-18T16:46:12Z
author_association: MEMBER

PR comments:
user: login: mergify[bot]
created_at: 2025-03-13T22:22:21Z
body: Cherry-pick of 0d931a502aca19b74acd3e3375c7ca5d5422ff5d has failed:
```
On branch mergify/bp/8.16/pr-17310
Your branch is up to date with 'origin/8.16'.

You are currently cherry-picking commit 0d931a50.
  (fix conflicts and run "git cherry-pick --continue")
  (use "git cherry-pick --skip" to skip this patch)
  (use "git cherry-pick --abort" to cancel the cherry-pick operation)

Changes to be committed:
	modified:   rakelib/artifacts.rake

Unmerged paths:
  (use "git add <file>..." to mark resolution)
	both modified:   rubyUtils.gradle

```


To fix up this pull request, you can check it out locally. See documentation: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally
author_association: CONTRIBUTOR
user: login: elastic-sonarqube[bot]
created_at: 2025-03-14T02:22:59Z
body: ## [![Quality Gate passed](https://raw.githubusercontent.com/SonarSource/sonarqube-static-resources/master/v104/checks/QualityGateBadge/passed-16px.png 'Quality Gate passed')](https://sonar.elastic.dev/dashboard?id=elastic_logstash_AYm_nEbQaV3I-igkX1q9&pullRequest=17316) **Quality Gate passed**  
Issues  
[![](https://raw.githubusercontent.com/SonarSource/sonarqube-static-resources/master/v104/common/passed-16px.png '')](https://sonar.elastic.dev/project/issues?id=elastic_logstash_AYm_nEbQaV3I-igkX1q9&pullRequest=17316&resolved=false) [0 New issues](https://sonar.elastic.dev/project/issues?id=elastic_logstash_AYm_nEbQaV3I-igkX1q9&pullRequest=17316&resolved=false)  
[![](https://raw.githubusercontent.com/SonarSource/sonarqube-static-resources/master/v104/common/fixed-16px.png '')](https://sonar.elastic.dev/project/issues?id=elastic_logstash_AYm_nEbQaV3I-igkX1q9&fixedInPullRequest=17316) [0 Fixed issues](https://sonar.elastic.dev/project/issues?id=elastic_logstash_AYm_nEbQaV3I-igkX1q9&fixedInPullRequest=17316)  
[![](https://raw.githubusercontent.com/SonarSource/sonarqube-static-resources/master/v104/common/accepted-16px.png '')](https://sonar.elastic.dev/project/issues?id=elastic_logstash_AYm_nEbQaV3I-igkX1q9&pullRequest=17316&issueStatuses=ACCEPTED) [0 Accepted issues](https://sonar.elastic.dev/project/issues?id=elastic_logstash_AYm_nEbQaV3I-igkX1q9&pullRequest=17316&issueStatuses=ACCEPTED)

Measures  
[![](https://raw.githubusercontent.com/SonarSource/sonarqube-static-resources/master/v104/common/passed-16px.png '')](https://sonar.elastic.dev/security_hotspots?id=elastic_logstash_AYm_nEbQaV3I-igkX1q9&pullRequest=17316) [0 Security Hotspots](https://sonar.elastic.dev/security_hotspots?id=elastic_logstash_AYm_nEbQaV3I-igkX1q9&pullRequest=17316)  
![](https://raw.githubusercontent.com/SonarSource/sonarqube-static-resources/master/v104/common/no-data-16px.png '') No data about Coverage  
![](https://raw.githubusercontent.com/SonarSource/sonarqube-static-resources/master/v104/common/no-data-16px.png '') No data about Duplication  
  
[See analysis details on SonarQube](https://sonar.elastic.dev/dashboard?id=elastic_logstash_AYm_nEbQaV3I-igkX1q9&pullRequest=17316)


author_association: NONE
user: login: elasticmachine
created_at: 2025-03-14T02:37:32Z
body: ## :green_heart: Build Succeeded
* [Buildkite Build](https://buildkite.com/elastic/logstash-pull-request-pipeline/builds/2619)
* Commit: c7d8593897c5580cda2406a0c208b6e33b1e3f89

### History
* :broken_heart: [Build #2607](https://buildkite.com/elastic/logstash-pull-request-pipeline/builds/2607) failed 086c93d36e511a2aa332749537e2f384ed435a34

cc @donoghuc

<!--buildkite-pr-comment-logstash-pull-request-pipeline
{"builds":[{"buildStatus":{"state":"passed","success":true,"hasRetries":false,"hasNonPreemptionRetries":false},"url":"https://buildkite.com/elastic/logstash-pull-request-pipeline/builds/2619","number":2619,"commit":"c7d8593897c5580cda2406a0c208b6e33b1e3f89"},{"buildStatus":{"state":"failed","success":false,"hasRetries":true,"hasNonPreemptionRetries":true},"url":"https://buildkite.com/elastic/logstash-pull-request-pipeline/builds/2607","number":2607,"commit":"086c93d36e511a2aa332749537e2f384ed435a34"}],"number":2619}
buildkite-pr-comment-->
author_association: NONE
user: login: mergify[bot]
created_at: 2025-03-17T05:01:53Z
body: This pull request has not been merged yet. Could you please review and merge it @donoghuc? 🙏
author_association: CONTRIBUTOR