Repository: kestra-io/kestra
Issue: [SCRIPT 🐍] `LocalFiles` task
Labels: backend

### Feature description

_Part of the the Script Workflow revamp_

`LocalFiles` is a task that let users write "inline" files in Kestra. It can used in `WorkingDirectory` or to pass complex files to other tasks (see DSL and example).
Also it can expose files to Kestra Outputs (see DSL and example).

## DSL

```yaml
type: LocalFiles
inputs:
  [Array of files] accessible from {{ outputs.<taskid>.inputs.<filename>}}
outputs:
  [Array of files] accessible from {{ outputs.<taskid>.outputs.<filename> }}
```

## Examples

1. Inside a `WorkingDirectory` - inputs are accessible directly as part of the working directory
```yaml
id: working_directory
type: WorkingDirectory
tasks:
  - id: local_file
    type: LocalFile
    inputs:
      main.py: |
        print("Hello World!")

  - id: run_python
    type: RunPython
    commands:
      - python main.py
```

2. Through outputs

```yaml
  - id: local_file
    type: LocalFile
    inputs:
      main.py: |
        print("Hello World!")

  - id: run_python
    type: RunPython
    commands:
      - python {{ outputs.local_file.inputs['main.py'] }}
```

3. Exposing files.
In this example a Python task create a files inside a `WorkingDirectory`then outputs are exposed through `outputDir` 

```yaml
id: working_directory
type: WorkingDirectory
tasks:

  - id: run_python
    type: RunPython
    commands:
      - pip install requests pandas
    script: |
      import requests
      import pandas as pd
      import io
      data = requests.get("https://raw.githubusercontent.com/cs109/2014_data/master/countries.csv").content
      df = pd.read_csv(io.StringIO(data.decode('utf-8')))
      df.assign(d='2023-06-01').to_csv("{{outputDir()}}/processed_data.csv", index=False)
```
Here data is written to the task outputDir in a file names "processed_data.csv" (so all files written in `outputDir` are exposed in outputs, so users could retrieve them with `outputs.my_task.outputDir["data.csv"]` for example)


---

💡 This feature could be directly linked to the a Multifile Editor feature. We could imagine each time the user click on the task, the right pane open a full editor where he can edit his files smoothly.



Issue comments:
I would not add a localfile to an output as outputs are not meant to store large piece of data.
My ideas was:
- `inputFiles` allow to create a file inside the local filesystem
- `outputFiles` allow to send a local file to Kestra's internal storage

OutputFiles will still be usable in other tasks that handle Kestra's internal storage files.
In fact we may need three things:
- A way to create a local file (inline file)
- A way to create a file inside the internal storage (inline file)
- A way to send an existing local file inside the internal storage

Pull request review comments:
I think it is missing a check on what was actually created in storage from the inputs as for eg. execution.txt contains a wrong pebble expression (toto instead of inputs.toto) ?
Same for storage file as content which should result in a duplicate content inside the newly created file ?
Assert the error message
Assert the error message

Patch that resolved the issue:
diff --git a/core/src/main/java/io/kestra/core/tasks/storages/LocalFiles.java b/core/src/main/java/io/kestra/core/tasks/storages/LocalFiles.java
new file mode 100644
index 0000000000..a2eb25ff18
--- /dev/null
+++ b/core/src/main/java/io/kestra/core/tasks/storages/LocalFiles.java
@@ -0,0 +1,168 @@
+package io.kestra.core.tasks.storages;
+
+import io.kestra.core.exceptions.IllegalVariableEvaluationException;
+import io.kestra.core.models.annotations.Example;
+import io.kestra.core.models.annotations.Plugin;
+import io.kestra.core.models.annotations.PluginProperty;
+import io.kestra.core.models.tasks.Output;
+import io.kestra.core.models.tasks.RunnableTask;
+import io.kestra.core.models.tasks.Task;
+import io.kestra.core.runners.RunContext;
+import io.kestra.core.tasks.scripts.BashService;
+import io.kestra.core.utils.ListUtils;
+import io.swagger.v3.oas.annotations.media.Schema;
+import lombok.*;
+import lombok.experimental.SuperBuilder;
+import org.apache.commons.io.IOUtils;
+import org.slf4j.Logger;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.PathMatcher;
+import java.util.AbstractMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static io.kestra.core.utils.Rethrow.throwBiConsumer;
+import static io.kestra.core.utils.Rethrow.throwFunction;
+
+@SuperBuilder
+@ToString
+@EqualsAndHashCode
+@Getter
+@NoArgsConstructor
+@Schema(
+    title = "Allow to create files in the local filesystem or to send files from the local filesystem to the internal storage.",
+    description = "This task should be used with the WorkingDirectory task to be able to access the same local filesystem within multiple tasks."
+)
+@Plugin(examples = {
+    @Example(
+        full = true,
+        title = "Create a local file that will be accessible to a bash task",
+        code = """
+            id: "local-files"
+            namespace: "io.kestra.tests"
+
+            tasks:
+              - id: workingDir
+                type: io.kestra.core.tasks.flows.WorkingDirectory
+                tasks:
+                - id: inputFiles
+                  type: io.kestra.core.tasks.storages.LocalFiles
+                  inputs:
+                    hello.txt: "Hello World\\n"
+                    addresse.json: "{{ outputs.myTaskId.uri }}"
+                - id: bash
+                  type: io.kestra.core.tasks.scripts.Bash
+                  commands:
+                    - cat hello.txt
+            """
+    ),
+    @Example(
+        full = true,
+        title = "Send local files to Kestra's internal storage",
+        code = """
+            id: "local-files"
+            namespace: "io.kestra.tests"
+
+            tasks:
+              - id: workingDir
+                type: io.kestra.core.tasks.flows.WorkingDirectory
+                tasks:
+                - id: bash
+                  type: io.kestra.core.tasks.scripts.Bash
+                  commands:
+                    - mkdir -p sub/dir
+                    - echo "Hello from Bash" >> sub/dir/bash1.txt
+                    - echo "Hello from Bash" >> sub/dir/bash2.txt
+                - id: outputFiles
+                  type: io.kestra.core.tasks.storages.LocalFiles
+                  outputs:
+                    - sub/**
+            """
+    )
+})
+public class LocalFiles extends Task implements RunnableTask<LocalFiles.LocalFilesOutput> {
+    @Schema(title = "The files to create on the local filesystem")
+    @PluginProperty(dynamic = true)
+    private Object inputs;
+
+    @Schema(
+        title = "The files from the local filesystem to send to the internal storage",
+        description = "must be a [Glob expression](https://en.wikipedia.org/wiki/Glob_(programming)) relative to current working directory, some examples: `my-dir/**`, `my-dir/*/**` or `my-dir/my-file.txt`"
+    )
+    @PluginProperty(dynamic = true)
+    private List<String> outputs;
+
+    @Override
+    public LocalFilesOutput run(RunContext runContext) throws Exception {
+        Logger logger = runContext.logger();
+
+        Map<String, String> inputFiles = this.inputs == null ? Map.of() : BashService.transformInputFiles(runContext, this.inputs);
+
+        inputFiles
+            .forEach(throwBiConsumer((fileName, input) -> {
+                var file = new File(runContext.tempDir().toString(), fileName);
+                if (file.exists()) {
+                    throw new IllegalVariableEvaluationException("File '" + fileName + "' already exist!");
+                }
+
+                if (!file.getParentFile().exists()) {
+                    //noinspection ResultOfMethodCallIgnored
+                    file.getParentFile().mkdirs();
+                }
+
+                var fileContent = runContext.render(input);
+                if (fileContent.startsWith("kestra://")) {
+                    try (var is = runContext.uriToInputStream(URI.create(fileContent));
+                         var out = new FileOutputStream(file)) {
+                        IOUtils.copyLarge(is, out);
+                    }
+                } else {
+                    Files.write(file.toPath(), fileContent.getBytes());
+                }
+            }));
+
+        var outputFiles = ListUtils.emptyOnNull(outputs)
+            .stream()
+            .flatMap(throwFunction(output -> this.outputMatcher(runContext, output)))
+            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+        logger.info("Provide {} input(s) and capture {} output(s).", inputFiles.size(), outputFiles.size());
+
+        return LocalFilesOutput.builder()
+            .uris(outputFiles)
+            .build();
+    }
+
+    private Stream<AbstractMap.SimpleEntry<String, URI>> outputMatcher(RunContext runContext, String output) throws IllegalVariableEvaluationException, IOException {
+        var glob = runContext.render(output);
+        final PathMatcher pathMatcher = FileSystems.getDefault().getPathMatcher("glob:" + glob);
+
+        try (Stream<Path> walk = Files.walk(runContext.tempDir())) {
+            return walk
+                .filter(Files::isRegularFile)
+                .filter(path -> pathMatcher.matches(runContext.tempDir().relativize(path)))
+                .map(throwFunction(path -> new AbstractMap.SimpleEntry<>(
+                    runContext.tempDir().relativize(path).toString(),
+                    runContext.putTempFile(path.toFile())
+                )))
+                .toList()
+                .stream();
+        }
+    }
+
+    @Builder
+    @Getter
+    public static class LocalFilesOutput implements Output {
+        @Schema(title = "The URI of the files that have been sent to the internal storage")
+        private Map<String, URI> uris;
+    }
+}


Test patch:
diff --git a/core/src/test/java/io/kestra/core/tasks/storages/LocalFilesTest.java b/core/src/test/java/io/kestra/core/tasks/storages/LocalFilesTest.java
new file mode 100644
index 0000000000..600b76961e
--- /dev/null
+++ b/core/src/test/java/io/kestra/core/tasks/storages/LocalFilesTest.java
@@ -0,0 +1,131 @@
+package io.kestra.core.tasks.storages;
+
+import io.kestra.core.exceptions.IllegalVariableEvaluationException;
+import io.kestra.core.runners.RunContextFactory;
+import io.kestra.core.storages.StorageInterface;
+import io.kestra.core.utils.IdUtils;
+import io.micronaut.test.extensions.junit5.annotation.MicronautTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Test;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+@MicronautTest
+class LocalFilesTest {
+    @Inject
+    RunContextFactory runContextFactory;
+
+    @Inject
+    StorageInterface storageInterface;
+
+    private URI internalFiles() throws IOException, URISyntaxException {
+        var resource = ConcatTest.class.getClassLoader().getResource("application.yml");
+
+        return storageInterface.put(
+            new URI("/file/storage/get.yml"),
+            new FileInputStream(Objects.requireNonNull(resource).getFile())
+        );
+    }
+
+
+    @Test
+    void run() throws Exception {
+        var runContext = runContextFactory.of(Map.of("toto", "tata"));
+        var storageFile = internalFiles();
+
+        var task = LocalFiles.builder()
+            .id(IdUtils.create())
+            .type(LocalFiles.class.getName())
+            .inputs(Map.of(
+                "hello-input.txt", "Hello Input",
+                "execution.txt", "{{toto}}",
+                "application.yml", storageFile.toString()
+            ))
+            .outputs(List.of("hello-input.txt"))
+            .build();
+        var outputs = task.run(runContext);
+
+        assertThat(outputs, notNullValue());
+        assertThat(outputs.getUris(), notNullValue());
+        assertThat(outputs.getUris().size(), is(1));
+        assertThat(
+            new String(storageInterface.get(outputs.getUris().get("hello-input.txt")).readAllBytes()),
+            is("Hello Input")
+        );
+        assertThat(runContext.tempDir().toFile().list().length, is(2));
+        assertThat(Files.readString(runContext.tempDir().resolve("execution.txt")), is("tata"));
+        assertThat(
+            Files.readString(runContext.tempDir().resolve("application.yml")),
+            is(new String(storageInterface.get(storageFile).readAllBytes()))
+        );
+
+        runContext.cleanup();
+    }
+
+    @Test
+    void recursive() throws Exception {
+        var runContext = runContextFactory.of(Map.of("toto", "tata"));
+        var storageFile = internalFiles();
+
+        var task = LocalFiles.builder()
+            .id(IdUtils.create())
+            .type(LocalFiles.class.getName())
+            .inputs(Map.of(
+                "test/hello-input.txt", "Hello Input",
+                "test/sub/dir/2/execution.txt", "{{toto}}",
+                "test/sub/dir/3/application.yml", storageFile.toString()
+            ))
+            .outputs(List.of("test/**"))
+            .build();
+        var outputs = task.run(runContext);
+
+        assertThat(outputs, notNullValue());
+        assertThat(outputs.getUris(), notNullValue());
+        assertThat(outputs.getUris().size(), is(3));
+        assertThat(
+            new String(storageInterface.get(outputs.getUris().get("test/hello-input.txt")).readAllBytes()),
+            is("Hello Input")
+        );
+        assertThat(
+            new String(storageInterface.get(outputs.getUris().get("test/sub/dir/2/execution.txt"))
+                .readAllBytes()),
+            is("tata")
+        );
+        assertThat(
+            new String(storageInterface.get(outputs.getUris().get("test/sub/dir/3/application.yml"))
+                .readAllBytes()),
+            is(new String(storageInterface.get(storageFile).readAllBytes()))
+        );
+        runContext.cleanup();
+    }
+
+    @Test
+    void failWithExistingInputFile() throws IOException {
+        var runContext = runContextFactory.of();
+        Files.createFile(Path.of(runContext.tempDir().toString(), "hello-input.txt"));
+
+        var task = LocalFiles.builder()
+            .id(IdUtils.create())
+            .type(LocalFiles.class.getName())
+            .inputs(Map.of(
+                "hello-input.txt", "Hello Input",
+                "execution.txt", "{{toto}}"
+            ))
+            .build();
+
+        assertThrows(IllegalVariableEvaluationException.class, () -> task.run(runContext));
+    }
+}
diff --git a/jdbc/src/test/java/io/kestra/jdbc/runner/JdbcRunnerTest.java b/jdbc/src/test/java/io/kestra/jdbc/runner/JdbcRunnerTest.java
index 51ae829afb..ba2253d3fb 100644
--- a/jdbc/src/test/java/io/kestra/jdbc/runner/JdbcRunnerTest.java
+++ b/jdbc/src/test/java/io/kestra/jdbc/runner/JdbcRunnerTest.java
@@ -180,7 +180,7 @@ void withTemplate() throws Exception {
         TemplateTest.withTemplate(runnerUtils, templateRepository, repositoryLoader, logsQueue);
     }
 
-    @RetryingTest(5)
+    @RetryingTest(5) // flaky on MySQL
     void withFailedTemplate() throws Exception {
         TemplateTest.withFailedTemplate(runnerUtils, templateRepository, repositoryLoader, logsQueue);
     }