Repository: kestra-io/kestra
Issue: Add Secret environment variable
Labels: backend

### Feature description
Add a `SECRET` environment variable with base64 encodage letting user add simple secret variables in configuration

Issue comments:

Pull request review comments:
You should get all env var at bean creation time and prepare a map of key/value, this would be more performant than calling System.getenv at each function call.
:+1: 
`SECRETS_` should be defined as a constant.
Should be toLowerCase I think, and you can use the length of the prefix constant instead of hardcoding 8 ;) 
I get them as uppercase because env var will probably be as uppercase when set up. Whatever I enforce both the map input and the get as uppercase for now, is it ok ?

Patch that resolved the issue:
diff --git a/build.gradle b/build.gradle
index c37ecf0c06..6387373985 100644
--- a/build.gradle
+++ b/build.gradle
@@ -168,6 +168,8 @@ subprojects {
         // configure en_US default locale for tests
         systemProperty 'user.language', 'en'
         systemProperty 'user.country', 'US'
+
+        environment 'SECRETS_MY_SECRET', "{\"secretKey\":\"secretValue\"}".bytes.encodeBase64().toString()
     }
 
     testlogger {
diff --git a/core/src/main/java/io/kestra/core/runners/pebble/Extension.java b/core/src/main/java/io/kestra/core/runners/pebble/Extension.java
index 52ab79452c..d06d8956d3 100644
--- a/core/src/main/java/io/kestra/core/runners/pebble/Extension.java
+++ b/core/src/main/java/io/kestra/core/runners/pebble/Extension.java
@@ -1,5 +1,6 @@
 package io.kestra.core.runners.pebble;
 
+import io.kestra.core.runners.pebble.functions.SecretFunction;
 import io.pebbletemplates.pebble.extension.*;
 import io.pebbletemplates.pebble.operator.Associativity;
 import io.pebbletemplates.pebble.operator.BinaryOperator;
@@ -17,12 +18,17 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+
+import jakarta.inject.Inject;
 import jakarta.inject.Singleton;
 
 import static io.pebbletemplates.pebble.operator.BinaryOperatorType.NORMAL;
 
 @Singleton
 public class Extension extends AbstractExtension {
+    @Inject
+    private SecretFunction secretFunction;
+
     @Override
     public List<TokenParser> getTokenParsers() {
         return null;
@@ -82,6 +88,7 @@ public Map<String, Function> getFunctions() {
         tests.put("now", new NowFunction());
         tests.put("json", new JsonFunction());
         tests.put("currentEachOutput", new CurrentEachOutputFunction());
+        tests.put("secret", secretFunction);
 
         return tests;
     }
diff --git a/core/src/main/java/io/kestra/core/runners/pebble/functions/SecretFunction.java b/core/src/main/java/io/kestra/core/runners/pebble/functions/SecretFunction.java
new file mode 100644
index 0000000000..19c4c6c836
--- /dev/null
+++ b/core/src/main/java/io/kestra/core/runners/pebble/functions/SecretFunction.java
@@ -0,0 +1,46 @@
+package io.kestra.core.runners.pebble.functions;
+
+import io.kestra.core.exceptions.IllegalVariableEvaluationException;
+import io.kestra.core.secret.SecretService;
+import io.pebbletemplates.pebble.error.PebbleException;
+import io.pebbletemplates.pebble.extension.Function;
+import io.pebbletemplates.pebble.template.EvaluationContext;
+import io.pebbletemplates.pebble.template.PebbleTemplate;
+import jakarta.inject.Inject;
+import jakarta.inject.Singleton;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+@Singleton
+public class SecretFunction implements Function {
+    @Inject
+    private SecretService secretService;
+
+    @Override
+    public List<String> getArgumentNames() {
+        return List.of("key");
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    public Object execute(Map<String, Object> args, PebbleTemplate self, EvaluationContext context, int lineNumber) {
+        String key = getSecretKey(args, self, lineNumber);
+        Map<String, String> flow = (Map<String, String>) context.getVariable("flow");
+
+        try {
+            return secretService.findSecret(flow.get("namespace"), key);
+        } catch (IllegalVariableEvaluationException | IOException e) {
+            throw new PebbleException(e, e.getMessage(), lineNumber, self.getName());
+        }
+    }
+
+    protected String getSecretKey(Map<String, Object> args, PebbleTemplate self, int lineNumber) {
+        if (!args.containsKey("key")) {
+            throw new PebbleException(null, "The 'secret' function expects an argument 'key'.", lineNumber, self.getName());
+        }
+
+        return (String) args.get("key");
+    }
+}
\ No newline at end of file
diff --git a/core/src/main/java/io/kestra/core/secret/SecretService.java b/core/src/main/java/io/kestra/core/secret/SecretService.java
new file mode 100644
index 0000000000..13ba7f97aa
--- /dev/null
+++ b/core/src/main/java/io/kestra/core/secret/SecretService.java
@@ -0,0 +1,34 @@
+package io.kestra.core.secret;
+
+import io.kestra.core.exceptions.IllegalVariableEvaluationException;
+import jakarta.inject.Singleton;
+
+import javax.annotation.PostConstruct;
+import java.io.IOException;
+import java.util.Base64;
+import java.util.Map;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+@Singleton
+public class SecretService {
+    private static final String SECRET_PREFIX = "SECRETS_";
+    private Map<String, String> decodedSecrets;
+
+    @PostConstruct
+    private void postConstruct() {
+        decodedSecrets = System.getenv().entrySet().stream().filter(entry -> entry.getKey().startsWith(SECRET_PREFIX)).collect(Collectors.toMap(
+            entry -> entry.getKey().substring(SECRET_PREFIX.length()).toUpperCase(),
+            entry -> new String(Base64.getDecoder().decode(entry.getValue()))
+        ));
+    }
+
+    public String findSecret(String namespace, String key) throws IOException, IllegalVariableEvaluationException {
+        return Optional
+            .ofNullable(decodedSecrets.get(key.toUpperCase()))
+            .orElseThrow(() -> new IllegalVariableEvaluationException("Unable to find secret '" + key + "'. " +
+                "You should add it in your environment variables as 'SECRETS_" + key.toUpperCase() +
+                "' with base64-encoded value."
+            ));
+    }
+}


Test patch:
diff --git a/core/src/test/java/io/kestra/core/secret/SecretFunctionTest.java b/core/src/test/java/io/kestra/core/secret/SecretFunctionTest.java
new file mode 100644
index 0000000000..9e30fd8329
--- /dev/null
+++ b/core/src/test/java/io/kestra/core/secret/SecretFunctionTest.java
@@ -0,0 +1,39 @@
+package io.kestra.core.secret;
+
+import io.kestra.core.exceptions.IllegalVariableEvaluationException;
+import io.kestra.core.models.executions.Execution;
+import io.kestra.core.runners.AbstractMemoryRunnerTest;
+import io.kestra.core.runners.RunnerUtils;
+import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.concurrent.TimeoutException;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+@MicronautTest
+public class SecretFunctionTest extends AbstractMemoryRunnerTest {
+    @Inject
+    private RunnerUtils runnerUtils;
+
+    @Inject
+    private SecretService secretService;
+
+    @Test
+    void getSecret() throws TimeoutException {
+        Execution execution = runnerUtils.runOne("io.kestra.tests", "secrets");
+        assertThat(execution.getTaskRunList().get(0).getOutputs().get("value"), is("secretValue"));
+    }
+
+    @Test
+    void getUnknownSecret() {
+        IllegalVariableEvaluationException exception = Assertions.assertThrows(IllegalVariableEvaluationException.class, () ->
+            secretService.findSecret(null, "unknown_secret_key")
+        );
+
+        assertThat(exception.getMessage(), is("Unable to find secret 'unknown_secret_key'"));
+    }
+}
diff --git a/core/src/test/resources/flows/valids/secrets.yaml b/core/src/test/resources/flows/valids/secrets.yaml
new file mode 100644
index 0000000000..2f5108bd35
--- /dev/null
+++ b/core/src/test/resources/flows/valids/secrets.yaml
@@ -0,0 +1,7 @@
+id: secrets
+namespace: io.kestra.tests
+
+tasks:
+  - id: get-secret
+    type: io.kestra.core.tasks.debugs.Return
+    format: "{{json(secret('my_secret')).secretKey}}"
\ No newline at end of file