internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: PrefectHQ/prefect

Issue:
DockerEnvironment push/pull needs newline
Can print nothing to just move the cursor down a single line so output doesn't overwrite push/pull progress bar


Discussion:

Resolving patch:
diff --git a/src/prefect/environments/docker.py b/src/prefect/environments/docker.py
--- a/src/prefect/environments/docker.py
+++ b/src/prefect/environments/docker.py
@@ -157,6 +157,7 @@ def pull_image(self) -> None:
         for line in output:
             if line.get("progress"):
                 print(line.get("status"), line.get("progress"), end="\r")
+        print("")
 
     def push_image(self, image_name: str, image_tag: str) -> None:
         """Push this environment to a registry
@@ -173,6 +174,7 @@ def push_image(self, image_name: str, image_tag: str) -> None:
         for line in output:
             if line.get("progress"):
                 print(line.get("status"), line.get("progress"), end="\r")
+        print("")
 
     def create_dockerfile(self, flow: "prefect.Flow", directory: str = None) -> None:
         """Creates a dockerfile to use as the container.


Test patch:
Repository: PrefectHQ/prefect

Issue:
Context docs are broken
For some reason the actual `context` class signature is not being documented.


Discussion:

Resolving patch:
diff --git a/src/prefect/tasks/templates/jinja2.py b/src/prefect/tasks/templates/jinja2.py
--- a/src/prefect/tasks/templates/jinja2.py
+++ b/src/prefect/tasks/templates/jinja2.py
@@ -6,6 +6,7 @@
 
 import prefect
 from prefect import Task
+from prefect.utilities.tasks import defaults_from_attrs
 
 
 class JinjaTemplateTask(Task):
@@ -23,9 +24,10 @@ class JinjaTemplateTask(Task):
     """
 
     def __init__(self, template: str = None, **kwargs: Any):
-        self.template = Template(template or "")
+        self.template = template or ""
         super().__init__(**kwargs)
 
+    @defaults_from_attrs("template")
     def run(self, template: str = None, **format_kwargs: Any) -> str:  # type: ignore
         """
         Formats the Jinja Template with the provided kwargs.
@@ -39,6 +41,6 @@ def run(self, template: str = None, **format_kwargs: Any) -> str:  # type: ignor
         Returns:
             - str: the rendered string
         """
-        template = self.template if template is None else Template(template)
+        template = Template(template)
         with prefect.context(**format_kwargs) as data:
             return template.render(**data)
diff --git a/src/prefect/utilities/context.py b/src/prefect/utilities/context.py
--- a/src/prefect/utilities/context.py
+++ b/src/prefect/utilities/context.py
@@ -10,9 +10,11 @@
 
 ```python
 import prefect.context
+
 with prefect.context(a=1, b=2):
     print(prefect.context.a) # 1
-print (prefect.context.a) # undefined
+
+print(prefect.context.a) # undefined
 ```
 
 Prefect provides various key / value pairs in context that are always available during task runs:
@@ -28,6 +30,8 @@
 | `tomorrow` | tomorrow's date formatted as `YYYY-MM-DD`|
 | `tomorrow_nodash` | tomorrow's date formatted as `YYYYMMDD`|
 | `task_name` | the name of the current task |
+
+Users can also provide values to context at runtime.
 """
 
 import contextlib


Test patch:
diff --git a/tests/tasks/test_strings.py b/tests/tasks/test_strings.py
--- a/tests/tasks/test_strings.py
+++ b/tests/tasks/test_strings.py
@@ -1,3 +1,4 @@
+import cloudpickle
 import os
 import pendulum
 import subprocess
@@ -93,3 +94,11 @@ def test_jinja_template_can_execute_python_code():
 
     assert res.is_successful()
     assert res.result[task].result == "1986-20 is a date."
+
+
+def test_jinja_task_is_pickleable():
+    task = JinjaTemplateTask(template="string")
+    new = cloudpickle.loads(cloudpickle.dumps(task))
+
+    assert isinstance(new, JinjaTemplateTask)
+    assert new.template == "string"
Repository: PrefectHQ/prefect

Issue:
Flow should serialize its environment no matter what
Currently, `flow.serialize()` only serializes its environment if `build=True`.  I _believe_ this is a vestige of old environment behavior and can be updated so the behavior is consistent.


Discussion:
Endorse

Resolving patch:
diff --git a/src/prefect/core/flow.py b/src/prefect/core/flow.py
--- a/src/prefect/core/flow.py
+++ b/src/prefect/core/flow.py
@@ -1122,8 +1122,8 @@ def serialize(self, build: bool = False) -> dict:
         Creates a serialized representation of the flow.
 
         Args:
-            - build (bool, optional): if `True`, the flow's environment is built and included
-                in the serialized flow. Otherwise, the environment is not included.
+            - build (bool, optional): if `True`, the flow's environment is built
+                prior to serialization
 
         Returns:
             - dict representing the flow
@@ -1138,7 +1138,7 @@ def serialize(self, build: bool = False) -> dict:
                 flow=self
             )  # type: Optional[Environment]
         else:
-            environment = None
+            environment = self.environment
 
         serialized.update(
             schema(only=["environment"]).dump({"environment": environment})


Test patch:
diff --git a/tests/core/test_flow.py b/tests/core/test_flow.py
--- a/tests/core/test_flow.py
+++ b/tests/core/test_flow.py
@@ -1281,12 +1281,13 @@ def test_default_environment_is_local_environment(self):
         f = Flow()
         assert isinstance(f.environment, prefect.environments.LocalEnvironment)
 
-    def test_serialize_builds_environment(self):
+    def test_serialize_includes_environment(self):
         f = Flow(environment=prefect.environments.LocalEnvironment())
         s_no_build = f.serialize()
         s_build = f.serialize(build=True)
 
-        assert s_no_build["environment"] is None
+        assert s_no_build["environment"]["type"] == "LocalEnvironment"
+        assert s_no_build["environment"]["serialized_flow"] is None
         assert s_build["environment"]["type"] == "LocalEnvironment"
 
     def test_to_environment_file_writes_data(self):
Repository: PrefectHQ/prefect

Issue:
Some states are not documented in the API docs
e.g., `Submitted` states


Discussion:

Resolving patch:
diff --git a/src/prefect/engine/state.py b/src/prefect/engine/state.py
--- a/src/prefect/engine/state.py
+++ b/src/prefect/engine/state.py
@@ -328,6 +328,15 @@ class Queued(_MetaState):
     The `Queued` state is used to indicate that another state could not transition to a
     `Running` state for some reason, often a lack of available resources.
 
+    The `Queued` state should be initialized with another state, which it wraps. The
+    wrapped state is extracted at the beginning of a task run.
+
+    Args:
+        - message (string): a message for the state.
+        - result (Any, optional): Defaults to `None`.
+        - state (State): the `State` state that has been marked as
+            "queued".
+
     """
 
 


Test patch:
Repository: PrefectHQ/prefect

Issue:
Add `idempotency_key` argument to Client
`client.create_flow_run` should have an `idempotency_key` argument to better expose that piece of functionality


Discussion:

Resolving patch:
diff --git a/src/prefect/client/client.py b/src/prefect/client/client.py
--- a/src/prefect/client/client.py
+++ b/src/prefect/client/client.py
@@ -393,16 +393,24 @@ def create_project(self, project_name: str) -> str:
     def create_flow_run(
         self,
         flow_id: str,
+        context: dict = None,
         parameters: dict = None,
         scheduled_start_time: datetime.datetime = None,
+        idempotency_key: str = None,
     ) -> str:
         """
         Create a new flow run for the given flow id.  If `start_time` is not provided, the flow run will be scheduled to start immediately.
 
         Args:
             - flow_id (str): the id of the Flow you wish to schedule
+            - context (dict, optional): the run context
             - parameters (dict, optional): a dictionary of parameter values to pass to the flow run
             - scheduled_start_time (datetime, optional): the time to schedule the execution for; if not provided, defaults to now
+            - idempotency_key (str, optional): an idempotency key; if provided, this run will be cached for 24
+                hours. Any subsequent attempts to create a run with the same idempotency key
+                will return the ID of the originally created run (no new run will be created after the first).
+                An error will be raised if parameters or context are provided and don't match the original.
+                Each subsequent request will reset the TTL for 24 hours.
 
         Returns:
             - str: the ID of the newly-created flow run
@@ -418,6 +426,10 @@ def create_flow_run(
         inputs = dict(flowId=flow_id)
         if parameters is not None:
             inputs.update(parameters=parameters)  # type: ignore
+        if context is not None:
+            inputs.update(context=context)  # type: ignore
+        if idempotency_key is not None:
+            inputs.update(idempotencyKey=idempotency_key)  # type: ignore
         if scheduled_start_time is not None:
             inputs.update(
                 scheduledStartTime=scheduled_start_time.isoformat()


Test patch:
Repository: PrefectHQ/prefect

Issue:
Upgrade `pytest`
We recently learned that the way Prefect uses `pytest` (specifically, some methods we call on markers in `conftest.py`) is incompatible with recent changes in `pytest >= 4.0`.  Just to keep things up-to-date, we should investigate this and bump / adjust `pytest` accordingly in `dev-requirements.txt`


Discussion:

Resolving patch:
diff --git a/conftest.py b/conftest.py
--- a/conftest.py
+++ b/conftest.py
@@ -93,7 +93,7 @@ def pytest_configure(config):
 
 
 def pytest_runtest_setup(item):
-    air_mark = item.get_marker("airflow")
+    air_mark = item.get_closest_marker("airflow")
 
     # if a test IS marked as "airflow" and the airflow flag IS NOT set, skip it
     if air_mark is not None and item.config.getoption("--airflow") is False:
@@ -105,7 +105,7 @@ def pytest_runtest_setup(item):
     elif air_mark is None and item.config.getoption("--airflow") is True:
         pytest.skip("Non-Airflow tests skipped because --airflow flag was provided.")
 
-    formatting_mark = item.get_marker("formatting")
+    formatting_mark = item.get_closest_marker("formatting")
 
     # if a test IS marked as "formatting" and the --skip-formatting flag IS set, skip it
     if (


Test patch:
diff --git a/tests/tasks/airtable/__init__.py b/tests/tasks/airtable/__init__.py
--- a/tests/tasks/airtable/__init__.py
+++ b/tests/tasks/airtable/__init__.py
@@ -1,3 +1,3 @@
 import pytest
 
-pytestmark = pytest.importorskip("airtable")
+pytest.importorskip("airtable")
diff --git a/tests/tasks/aws/__init__.py b/tests/tasks/aws/__init__.py
--- a/tests/tasks/aws/__init__.py
+++ b/tests/tasks/aws/__init__.py
@@ -1,3 +1,3 @@
 import pytest
 
-pytestmark = pytest.importorskip("boto3")
+pytest.importorskip("boto3")
diff --git a/tests/tasks/google/__init__.py b/tests/tasks/google/__init__.py
--- a/tests/tasks/google/__init__.py
+++ b/tests/tasks/google/__init__.py
@@ -1,3 +1,3 @@
 import pytest
 
-pytestmark = pytest.importorskip("google.cloud")
+pytest.importorskip("google.cloud")
diff --git a/tests/tasks/kubernetes/__init__.py b/tests/tasks/kubernetes/__init__.py
--- a/tests/tasks/kubernetes/__init__.py
+++ b/tests/tasks/kubernetes/__init__.py
@@ -1,3 +1,3 @@
 import pytest
 
-pytestmark = pytest.importorskip("kubernetes")
+pytest.importorskip("kubernetes")
diff --git a/tests/tasks/twitter/__init__.py b/tests/tasks/twitter/__init__.py
--- a/tests/tasks/twitter/__init__.py
+++ b/tests/tasks/twitter/__init__.py
@@ -1,3 +1,3 @@
 import pytest
 
-pytestmark = pytest.importorskip("tweepy")
+pytest.importorskip("tweepy")
Repository: PrefectHQ/prefect

Issue:
Coverage report link is broken
A 404 status code is returned when you try to navigate to the coverage report link.

Most likely caused by the changes in https://github.com/PrefectHQ/prefect/pull/805

**cc:** @jlowin 


Discussion:

Resolving patch:
diff --git a/docs/generate_docs.py b/docs/generate_docs.py
--- a/docs/generate_docs.py
+++ b/docs/generate_docs.py
@@ -293,7 +293,7 @@ def generate_coverage():
 
             # Unit test coverage report
 
-            To view the test coverage report, [click here](/prefect-coverage/index.html).
+            To view the test coverage report, <a href="/prefect-coverage/index.html">click here</a>.
             """
             ).lstrip()
         )


Test patch:
Repository: PrefectHQ/prefect

Issue:
flow.visualize() fails on windows
A simple flow.visualize() command fails when running on Windows.  Looks like it's due to the way that windows creates tempfiles (see this thread: https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file)

Traceback (most recent call last):
  File "C:\Dev\source\python\prefecttesting\dostuff.py", line 34, in <module>
    flow.visualize()
  File "C:\Dev\source\python\prefecttesting\venv\lib\site-packages\prefect\core\flow.py", line 1106, in visualize
    graph.render(tmp.name, view=True)
  File "C:\Dev\source\python\prefecttesting\venv\lib\site-packages\graphviz\files.py", line 183, in render
    filepath = self.save(filename, directory)
  File "C:\Dev\source\python\prefecttesting\venv\lib\site-packages\graphviz\files.py", line 155, in save
    with io.open(filepath, 'w', encoding=self.encoding) as fd:
PermissionError: [Errno 13] Permission denied: 'c:\\dev\\temp\\tmpn3cczj0a'


Discussion:
Thanks for the issue and the helpful pointer @mblye ! 

It looks like `graphviz` is attempting to open the temporary file a _second_ time here: https://github.com/PrefectHQ/prefect/blob/master/src/prefect/core/flow.py#L1105-L1106 and Windows doesn't like that. 

It should be possible to work around this by creating a temporary _directory_ and then using an arbitrary filename in the call to `graph.render` 👍 

Resolving patch:
diff --git a/src/prefect/core/flow.py b/src/prefect/core/flow.py
--- a/src/prefect/core/flow.py
+++ b/src/prefect/core/flow.py
@@ -3,6 +3,7 @@
 import functools
 import inspect
 import json
+import os
 import tempfile
 import time
 import uuid
@@ -1105,8 +1106,12 @@ def get_color(task: Task, map_index: int = None) -> str:
         except Exception:
             pass
 
-        with tempfile.NamedTemporaryFile() as tmp:
-            graph.render(tmp.name, view=True)
+        with tempfile.NamedTemporaryFile(delete=False) as tmp:
+            tmp.close()
+            try:
+                graph.render(tmp.name, view=True)
+            finally:
+                os.unlink(tmp.name)
 
         return graph
 


Test patch:
Repository: PrefectHQ/prefect

Issue:
Add hook for task logging from function tasks
Currently, if a user creates a task via class inheritance, adding additional custom logs is as simple as:
```python
class MyTask(Task):
   def run(self):
      self.logger.info("My own custom runtime log.")
```

However, if a user creates a task via the `@task` decorator, it's not obvious how to add runtime logs (as `self` isn't readily available).  We should:
- add a utility
- or add the logger to context
- or something else

to make custom logging easier.


Discussion:

Resolving patch:
diff --git a/src/prefect/engine/task_runner.py b/src/prefect/engine/task_runner.py
--- a/src/prefect/engine/task_runner.py
+++ b/src/prefect/engine/task_runner.py
@@ -783,9 +783,10 @@ def get_task_run_state(
             )
             timeout_handler = timeout_handler or main_thread_timeout
             raw_inputs = {k: r.value for k, r in inputs.items()}
-            result = timeout_handler(
-                self.task.run, timeout=self.task.timeout, **raw_inputs
-            )
+            with prefect.context(logger=self.task.logger):
+                result = timeout_handler(
+                    self.task.run, timeout=self.task.timeout, **raw_inputs
+                )
 
         # inform user of timeout
         except TimeoutError as exc:


Test patch:
diff --git a/tests/engine/test_task_runner.py b/tests/engine/test_task_runner.py
--- a/tests/engine/test_task_runner.py
+++ b/tests/engine/test_task_runner.py
@@ -1476,3 +1476,14 @@ def fn():
         upstream_states={Edge(Task(), Task()): Skipped()}
     )
     assert new_state.is_successful()
+
+
+def test_task_runner_provides_logger():
+    @prefect.task()
+    def my_task():
+        logger = prefect.context.get("logger")
+        return logger
+
+    state = TaskRunner(my_task).run()
+    assert state.is_successful()
+    assert state.result is my_task.logger
Repository: PrefectHQ/prefect

Issue:
Secrets are not secret within the Flow context manager, they should throw an error
While trying to figure out the best way to get some data into a deployed flow, I decided to use a secret. I thought it would be good to put my secret into the flow context manager so that I could pass it to my first task as a parameter:

```python
from prefect import Flow
from prefect.client import Secret

@task(name="example task")
def example_task(email, token):
  # does something

with Flow("Example Flow") as flow:
  email = Secret("email").get()
  token = Secret("token").get()
  example_task(email, token)
```

This is not secret at all, because the secrets simply become constants and can be easily seen in other parts of the system. So, prefect should throw an error if someone tries this.


Discussion:

Resolving patch:
diff --git a/src/prefect/client/secrets.py b/src/prefect/client/secrets.py
--- a/src/prefect/client/secrets.py
+++ b/src/prefect/client/secrets.py
@@ -38,8 +38,14 @@ def get(self) -> Optional[Any]:
             - Any: the value of the secret; if not found, returns `None`
 
         Raises:
+            - ValueError: if `.get()` is called within a Flow building context
             - ValueError: if `use_local_secrets=False` and the Client fails to retrieve your secret
         """
+        if isinstance(prefect.context.get("flow"), prefect.core.flow.Flow):
+            raise ValueError(
+                "Secrets should only be retrieved during a Flow run, not while building a Flow."
+            )
+
         if prefect.config.cloud.use_local_secrets is True:
             secrets = prefect.context.get("secrets", {})
             value = secrets.get(self.name)


Test patch:
diff --git a/tests/client/test_secrets.py b/tests/client/test_secrets.py
--- a/tests/client/test_secrets.py
+++ b/tests/client/test_secrets.py
@@ -59,3 +59,19 @@ def test_local_secrets_auto_load_json_strings():
         with prefect.context(secrets=dict(test='{"x": 42}')):
             assert secret.get() == {"x": 42}
         assert secret.get() is None
+
+
+def test_secrets_raise_if_in_flow_context():
+    secret = Secret(name="test")
+    with set_temporary_config({"cloud.use_local_secrets": True}):
+        with prefect.context(secrets=dict(test=42)):
+            with prefect.Flow("test"):
+                with pytest.raises(ValueError) as exc:
+                    secret.get()
+
+
+def test_secrets_dont_raise_just_because_flow_key_is_populated():
+    secret = Secret(name="test")
+    with set_temporary_config({"cloud.use_local_secrets": True}):
+        with prefect.context(secrets=dict(test=42), flow="not None"):
+            assert secret.get() == 42
Repository: PrefectHQ/prefect

Issue:
Decoding error when running a stored flow
```
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "/usr/local/lib/python3.6/site-packages/prefect/environments/execution/cloud/environment.py", line 104, in run_flow
    flow = schema.load(json.load(f))
  File "/usr/local/lib/python3.6/json/__init__.py", line 296, in load
    return loads(fp.read(),
  File "/usr/local/lib/python3.6/codecs.py", line 321, in decode
    (result, consumed) = self._buffer_decode(data, self.errors, final)
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte
```

Still investigating the issue


Discussion:

Resolving patch:
diff --git a/src/prefect/environments/execution/cloud/environment.py b/src/prefect/environments/execution/cloud/environment.py
--- a/src/prefect/environments/execution/cloud/environment.py
+++ b/src/prefect/environments/execution/cloud/environment.py
@@ -5,6 +5,7 @@
 from os import path
 from typing import Any, List
 
+import cloudpickle
 import docker
 import yaml
 
@@ -94,14 +95,13 @@ def run_flow(self) -> None:
             cluster.adapt(minimum=1, maximum=1)
 
             # Load serialized flow from file and run it with a DaskExecutor
-            schema = prefect.serialization.flow.FlowSchema()
             with open(
                 prefect.context.get(
                     "flow_file_path", "/root/.prefect/flow_env.prefect"
                 ),
-                "r",
+                "rb",
             ) as f:
-                flow = schema.load(json.load(f))
+                flow = cloudpickle.load(f)
 
                 executor = DaskExecutor(address=cluster.scheduler_address)
                 FlowRunner(flow=flow).run(executor=executor)


Test patch:
diff --git a/tests/environments/execution/test_cloud_environment.py b/tests/environments/execution/test_cloud_environment.py
--- a/tests/environments/execution/test_cloud_environment.py
+++ b/tests/environments/execution/test_cloud_environment.py
@@ -4,6 +4,7 @@
 from os import path
 from unittest.mock import MagicMock
 
+import cloudpickle
 import pytest
 import yaml
 
@@ -99,8 +100,8 @@ def test_run_flow(monkeypatch):
         with open(os.path.join(directory, "flow_env.prefect"), "w+") as env:
             flow = prefect.Flow("test")
             flow_path = os.path.join(directory, "flow_env.prefect")
-            with open(flow_path, "w") as f:
-                json.dump(flow.serialize(), f)
+            with open(flow_path, "wb") as f:
+                cloudpickle.dump(flow, f)
 
         with set_temporary_config({"cloud.auth_token": "test"}):
             with prefect.context(