internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: analysiscenter/batchflow

Issue:
Typo
https://github.com/analysiscenter/batchflow/blob/cd9062150811665b5a4e51a1080da5855f1c4dcb/batchflow/decorators.py#L337

`random_stat` -> `random_state`


Discussion:

Resolving patch:
diff --git a/batchflow/decorators.py b/batchflow/decorators.py
--- a/batchflow/decorators.py
+++ b/batchflow/decorators.py
@@ -334,7 +334,7 @@ def _make_args(self, iteration, init_args, args, kwargs, params=None):
         def make_random_seed(self):
             if getattr(self, 'random_state', None) is None:
                 return make_seed_sequence()
-            return self.random_stat
+            return self.random_state
 
         def wrap_with_threads(self, args, kwargs):
             """ Run a method in parallel threads """


Test patch:
Repository: analysiscenter/batchflow

Issue:
Test install for all supported Pythons: 3.8 - 3.12



Discussion:

Resolving patch:
diff --git a/batchflow/decorators.py b/batchflow/decorators.py
--- a/batchflow/decorators.py
+++ b/batchflow/decorators.py
@@ -485,10 +485,11 @@ def _jit(method):
             try:
                 func = make_function(method)
                 func = jit(*args, nopython=nopython, nogil=nogil, **kwargs)(func)
-            except Exception:
+            except Exception:   # pylint:disable=broad-except
                 # the source is not available or not compilable
                 func = method
-                logging.warning('The method cannot be compiled because the source code is not available. This causes a severe performance degradation for method %s',
+                logging.warning('The method cannot be compiled because the source code is not available. '
+                                'This causes a severe performance degradation for method %s',
                                 method.__name__)
         else:
             func = method


Test patch:
Repository: pfnet/pytorch-pfn-extras

Issue:
 Failed to rename a file in writing.py
The following error happens.

![image](https://user-images.githubusercontent.com/43694878/82112669-0ac22c80-978a-11ea-8cd6-55ee7b83691b.png)

![image](https://user-images.githubusercontent.com/43694878/82112676-27f6fb00-978a-11ea-8e99-cae7fe6caa79.png)

I fixed it.

![image](https://user-images.githubusercontent.com/43694878/82112652-bf0f8300-9789-11ea-9301-445881ecb682.png)



Discussion:

Resolving patch:
diff --git a/pytorch_pfn_extras/writing.py b/pytorch_pfn_extras/writing.py
--- a/pytorch_pfn_extras/writing.py
+++ b/pytorch_pfn_extras/writing.py
@@ -208,7 +208,7 @@ def save(self, filename, out_dir, target, savefun, **kwds):
         with self.fs.open(tmppath, 'wb') as f:
             # HDFS does not support overwrite
             savefun(target, f)
-            self.fs.rename(tmppath, dest)
+        self.fs.rename(tmppath, dest)
         if make_backup:
             self.fs.remove(bak)
 


Test patch:
Repository: pfnet/pytorch-pfn-extras

Issue:
[ONNX] Single input is not exported correctly
When exporting ONNX with single input, input tensor is split and input shape will be wrong.

https://github.com/pfnet/pytorch-pfn-extras/blob/2c5e1440f3137f48d651e705fddb7ea0251583d3/pytorch_pfn_extras/onnx/export_testcase.py#L177


Discussion:

Resolving patch:
diff --git a/pytorch_pfn_extras/onnx/export_testcase.py b/pytorch_pfn_extras/onnx/export_testcase.py
--- a/pytorch_pfn_extras/onnx/export_testcase.py
+++ b/pytorch_pfn_extras/onnx/export_testcase.py
@@ -163,6 +163,10 @@ def export_testcase(
     onnx_graph, outs = _export(
         model, args, strip_large_tensor_data, large_tensor_threshold,
         input_names=input_names, **kwargs)
+    if isinstance(args, torch.Tensor):
+        args = args,
+    if isinstance(outs, torch.Tensor):
+        outs = outs,
 
     # Remove unused inputs
     # - When keep_initializers_as_inputs=True, inputs contains initializers.
@@ -192,10 +196,6 @@ def write_to_pb(f, tensor, name=None):
                 _strip_raw_data(t)
             fp.write(t.SerializeToString())
 
-    if isinstance(args, torch.Tensor):
-        args = args,
-    if isinstance(outs, torch.Tensor):
-        outs = outs,
     data_set_path = os.path.join(out_dir, 'test_data_set_0')
     seq_id = 0
     while is_on_memory and os.path.exists(data_set_path):


Test patch:
diff --git a/tests/pytorch_pfn_extras_tests/onnx/test_export_testcase.py b/tests/pytorch_pfn_extras_tests/onnx/test_export_testcase.py
--- a/tests/pytorch_pfn_extras_tests/onnx/test_export_testcase.py
+++ b/tests/pytorch_pfn_extras_tests/onnx/test_export_testcase.py
@@ -69,7 +69,9 @@ def test_export_testcase():
     assert os.path.isfile(os.path.join(output_dir, 'meta.json'))
     assert os.path.isfile(os.path.join(output_dir, 'model.onnx'))
     test_data_set_dir = os.path.join(output_dir, 'test_data_set_0')
-    assert os.path.isfile(os.path.join(test_data_set_dir, 'input_0.pb'))
+    expected_input_0_path = os.path.join(test_data_set_dir, 'input_0.pb')
+    assert os.path.isfile(expected_input_0_path)
+    assert _to_array(expected_input_0_path).shape == (1, 1, 28, 28)
     assert os.path.isfile(os.path.join(test_data_set_dir, 'output_0.pb'))
     assert os.path.isfile(os.path.join(
         test_data_set_dir, 'gradient_input_0.pb'))
Repository: pfnet/pytorch-pfn-extras

Issue:
Create API reference website



Discussion:

Resolving patch:
diff --git a/docs/source/conf.py b/docs/source/conf.py
new file mode 100644
--- /dev/null
+++ b/docs/source/conf.py
@@ -0,0 +1,56 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'pytorch-pfn-extras'
+copyright = '2021, Preferred Networks, Inc.'
+author = 'Preferred Networks, Inc.'
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autosummary',
+    'myst_parser',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# Autosummary
+autosummary_generate = True
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'pydata_sphinx_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']


Test patch:
Repository: pfnet/pytorch-pfn-extras

Issue:
Make TensorBoardWriter no-op if tensorboard is not available



Discussion:

Resolving patch:
diff --git a/example/mnist.py b/example/mnist.py
--- a/example/mnist.py
+++ b/example/mnist.py
@@ -114,6 +114,12 @@ def main():
     # manager.extend(...) also works
     my_extensions = [
         extensions.LogReport(),
+
+        # Enables TensorBoard support.
+        # Run `tensorboard --logdir runs` to launch the TensorBoard.
+        extensions.LogReport(
+            writer=ppe.writing.TensorBoardWriter(out_dir='runs'),
+            trigger=(1, 'iteration')),
         extensions.ProgressBar(),
         extensions.observe_lr(optimizer=optimizer),
         extensions.ParameterStatistics(model, prefix='model'),
diff --git a/pytorch_pfn_extras/writing/_tensorboard_writer.py b/pytorch_pfn_extras/writing/_tensorboard_writer.py
--- a/pytorch_pfn_extras/writing/_tensorboard_writer.py
+++ b/pytorch_pfn_extras/writing/_tensorboard_writer.py
@@ -1,4 +1,5 @@
 from typing import Any, KeysView, Optional
+import warnings
 
 from pytorch_pfn_extras.writing._writer_base import (
     _TargetType, _SaveFun, _FileSystem
@@ -27,10 +28,17 @@ def __init__(
             stats: Optional[KeysView[str]] = None,
             **kwds: Any
     ) -> None:
-        from torch.utils.tensorboard import SummaryWriter
+        self._writer = None
+        try:
+            import torch.utils.tensorboard
+        except ImportError:
+            warnings.warn(
+                'tensorboard is unavailable. '
+                'TensorBoardWriter will do nothing.')
+            return
         self._stats = stats
-        self._writer: Optional[SummaryWriter] = (
-            SummaryWriter(  # type: ignore[no-untyped-call]
+        self._writer = (
+            torch.utils.tensorboard.SummaryWriter(  # type: ignore[no-untyped-call]
                 log_dir=out_dir, **kwds))
 
     def __del__(self) -> None:
@@ -57,8 +65,7 @@ def __call__(
             append: Ignored.
         """
         if self._writer is None:
-            raise RuntimeError('TensorBoardWriter already finalized')
-
+            return
         stats_cpu = target
         if isinstance(target, list):
             stats_cpu = target[-1]
@@ -74,7 +81,6 @@ def __call__(
                 key, value, stats_cpu['iteration'])
 
     def finalize(self) -> None:
-        writer = self._writer
-        if writer is not None:
-            writer.close()  # type: ignore[no-untyped-call]
-        self._writer = None
+        if self._writer is not None:
+            self._writer.close()  # type: ignore[no-untyped-call]
+            self._writer = None


Test patch:
Repository: pfnet/pytorch-pfn-extras

Issue:
Tests package is not correctly excluded
Currently test codes are installed to site-packages.

`setuptools.find_packages(exclude=['tests'])` needs to be fixed.


Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -20,6 +20,6 @@
         'onnx': ['onnx'],
     },
     python_requires='>=3.6.0',
-    packages=setuptools.find_packages(exclude=['tests']),
+    packages=setuptools.find_packages(exclude=['tests', 'tests.*']),
     package_data={'pytorch_pfn_extras': ['py.typed']},
 )


Test patch:
Repository: pfnet/pytorch-pfn-extras

Issue:
Typing: ManualScheduleTrigger `points` should accept `int`



Discussion:

Resolving patch:
diff --git a/pytorch_pfn_extras/training/triggers/manual_schedule_trigger.py b/pytorch_pfn_extras/training/triggers/manual_schedule_trigger.py
--- a/pytorch_pfn_extras/training/triggers/manual_schedule_trigger.py
+++ b/pytorch_pfn_extras/training/triggers/manual_schedule_trigger.py
@@ -1,6 +1,6 @@
 # mypy: ignore-errors
 
-from typing import List, Union, TYPE_CHECKING
+from typing import Sequence, Union, TYPE_CHECKING
 
 from pytorch_pfn_extras.training import trigger
 
@@ -29,7 +29,7 @@ class ManualScheduleTrigger(trigger.Trigger):
 
     """
 
-    def __init__(self, points: Union[float, List[float]], unit: 'UnitLiteral'):
+    def __init__(self, points: Union[float, Sequence[float]], unit: 'UnitLiteral'):
         if unit not in ('epoch', 'iteration'):
             raise ValueError(
                 'Trigger unit must be either \'epoch\' or \'iteration\'.')


Test patch:
Repository: pfnet/pytorch-pfn-extras

Issue:
pfto: Updates for torch 1.11



Discussion:

Resolving patch:
diff --git a/pytorch_pfn_extras/onnx/pfto_exporter/export.py b/pytorch_pfn_extras/onnx/pfto_exporter/export.py
--- a/pytorch_pfn_extras/onnx/pfto_exporter/export.py
+++ b/pytorch_pfn_extras/onnx/pfto_exporter/export.py
@@ -316,7 +316,11 @@ def optimize_onnx(self, graph: torch._C.Graph) -> torch._C.Graph:
         else:
             self.run_jit_pass(torch._C._jit_pass_onnx_scalar_type_analysis, graph)
 
-        if self.do_constant_folding and self.opset_version in torch.onnx.constant_folding_opset_versions:
+        opset_versions = (
+            sym_hel._constant_folding_opset_versions   # type: ignore[attr-defined]
+            if pytorch_pfn_extras.requires("1.11.0")
+            else torch.onnx.constant_folding_opset_versions)  # type: ignore[attr-defined]
+        if self.do_constant_folding and self.opset_version in opset_versions:
             folded: Dict[str, torch.IValue] = torch._C._jit_pass_onnx_constant_fold(  # type: ignore[attr-defined]
                 graph, self.vars, self.opset_version
             )
@@ -473,11 +477,15 @@ def symbolic_function(self, n: torch._C.Node) -> Optional[Callable]:
             # TODO(twata): Use repr(pyobj) in scope name or doc_string
             return cast(Callable, pyobj.symbolic)
         else:
+            domain = ""
             if ns == "prim":
-                op = f"prim_{op}"
-            if sym_reg.is_registered_op(op, "", self.opset_version):  # type: ignore[no-untyped-call]
+                if pytorch_pfn_extras.requires('1.11'):
+                    domain = "prim"
+                else:
+                    op = f"prim_{op}"
+            if sym_reg.is_registered_op(op, domain, self.opset_version):  # type: ignore[no-untyped-call]
                 return cast(
-                    Callable, sym_reg.get_registered_op(op, "", self.opset_version)  # type: ignore[no-untyped-call]
+                    Callable, sym_reg.get_registered_op(op, domain, self.opset_version)  # type: ignore[no-untyped-call]
                 )
             else:
                 return None


Test patch:
Repository: pfnet/pytorch-pfn-extras

Issue:
FP16 evaluator
Add an option to `ppe.training.Evaluator` to allow `autocast` to be applied.


Discussion:

Resolving patch:
diff --git a/pytorch_pfn_extras/handler/_logic.py b/pytorch_pfn_extras/handler/_logic.py
--- a/pytorch_pfn_extras/handler/_logic.py
+++ b/pytorch_pfn_extras/handler/_logic.py
@@ -372,7 +372,8 @@ def eval_step(
                 Input tensors feeded to the model of the current step.
         """
         model = models[self.model_name]
-        outs = self._forward(model, batch)
+        with self._autocast.autocast():
+            outs = self._forward(model, batch)
         return outs
 
 


Test patch:
diff --git a/tests/pytorch_pfn_extras_tests/training_tests/test_trainer.py b/tests/pytorch_pfn_extras_tests/training_tests/test_trainer.py
--- a/tests/pytorch_pfn_extras_tests/training_tests/test_trainer.py
+++ b/tests/pytorch_pfn_extras_tests/training_tests/test_trainer.py
@@ -763,20 +763,52 @@ def test_trainer_with_clousure_logic(device, progress_bar, path):
 
 
 @pytest.mark.gpu
-def test_trainer_with_autocast(path):
+@pytest.mark.parametrize("autocast_train", [True, False])
+@pytest.mark.parametrize("autocast_eval", [True, False])
+def test_trainer_with_autocast(path, autocast_train, autocast_eval):
     if not torch.cuda.is_available():
         pytest.skip()
-    model = MyModel()
+
+    class AutocastCheckModel(MyModel):
+        def __init__(self, autocast_train, autocast_eval):
+            super().__init__()
+            self.autocast_train = autocast_train
+            self.autocast_eval = autocast_eval
+
+        def forward(self, x):
+            if self.training:
+                assert torch.is_autocast_enabled() == self.autocast_train
+            if not self.training:
+                assert torch.is_autocast_enabled() == self.autocast_eval
+
+            return super().forward(x)
+
+    model = AutocastCheckModel(
+        autocast_train=autocast_train, autocast_eval=autocast_eval
+    )
     model_with_loss = MyModelWithLossFn(model)
     ppe.to(model_with_loss, "cuda")
     optimizer = torch.optim.SGD(model.parameters(), lr=0.1)
+    data = torch.utils.data.DataLoader(
+        [
+            {
+                "x": torch.rand(
+                    20,
+                ),
+                "t": torch.rand(
+                    10,
+                ),
+            }
+            for i in range(10)
+        ]
+    )
     extensions = []
-    autocast_options = {"autocast": True}
+
     evaluator = engine.create_evaluator(
-        model_with_loss, device="cuda", options=autocast_options
+        model_with_loss, device="cuda", options={"autocast": autocast_eval}
     )
 
-    engine.create_trainer(
+    trainer = engine.create_trainer(
         model_with_loss,
         optimizer,
         20,
@@ -784,5 +816,7 @@ def test_trainer_with_autocast(path):
         evaluator=evaluator,
         extensions=extensions,
         out_dir=path,
-        options=autocast_options,
+        options={"autocast": autocast_train},
     )
+
+    trainer.run(data, data)
Repository: pfnet/pytorch-pfn-extras

Issue:
`snapshot_on_error` in `DistributedSnapshot`
https://github.com/pfnet/pytorch-pfn-extras/blob/c5b4d58c16667d60e980ca96c5839416d63d50f2/pytorch_pfn_extras/training/extensions/_snapshot.py#L419-L428

If `snapshot_on_error` is specified in the snapshot extension, the `on_error()` method is called in case of an error, but in `DistributedSnapshot`, this method is not inherited and is not controlled by `saver_rank`.
This causes file corruption problems due to simultaneous writes, which needs to be corrected.


Discussion:

Resolving patch:
diff --git a/pytorch_pfn_extras/training/extensions/_snapshot.py b/pytorch_pfn_extras/training/extensions/_snapshot.py
--- a/pytorch_pfn_extras/training/extensions/_snapshot.py
+++ b/pytorch_pfn_extras/training/extensions/_snapshot.py
@@ -513,11 +513,10 @@ def _add_cleanup_hook(self, writer: writing.Writer) -> None:
         if self._rank == self._saver_rank:
             super()._add_cleanup_hook(writer)
 
-    def __call__(self, manager: ExtensionsManagerProtocol) -> None:
-        if self.condition():
-            # on distributed environments only the designed rank
-            # saves the snapshot
-            if self._rank == self._saver_rank:
-                self._make_snapshot(manager)
-            if self._size > 1:
-                torch.distributed.barrier()  # type: ignore[no-untyped-call]
+    def _make_snapshot(self, manager: ExtensionsManagerProtocol) -> None:
+        # on distributed environments only the designed rank
+        # saves the snapshot
+        if self._rank == self._saver_rank:
+            super()._make_snapshot(manager)
+        if self._size > 1:
+            torch.distributed.barrier()  # type: ignore


Test patch:
Repository: conan-io/conan

Issue:
Gcc detection fails sometimes
If gcc command does not exist:

Found gcc /bin/sh: gcc: command not found.0
CC and CXX: ,  
Default conan.conf settings
    os=Linux
    arch=x86_64
    compiler=gcc
    compiler.version=/bin/sh: gcc: command not found.0
    build_type=Release



Discussion:

Resolving patch:
diff --git a/conans/client/detect.py b/conans/client/detect.py
--- a/conans/client/detect.py
+++ b/conans/client/detect.py
@@ -25,7 +25,7 @@ def gcc_compiler(output):
     try:
         _, out = execute('gcc -dumpversion')
         compiler = "gcc"
-        installed_version = Version(out).minor(fill=False)
+        installed_version = re.search("([0-9]\.[0-9])", out).group()
         if installed_version:
             output.success("Found %s %s" % (compiler, installed_version))
             return compiler, installed_version


Test patch:
Repository: conan-io/conan

Issue:
mark headers as "SYSTEM" headers to silence warnings
Many libraries generate tons of warnings in public headers. WebSocket++ uses auto_ptr for example and many Boost libraries truncate integers implicitly (-Wconversion). To consume these libraries you have to treat them as system headers because GCC won't emit warnings in these.

This is how Conan currently sets the include directories:

``` CMake
include_directories(${CONAN_INCLUDE_DIRS})
```

This is how you would add them as "system" headers to silence warnings:

``` CMake
include_directories(SYSTEM ${CONAN_INCLUDE_DIRS})
```

Is there a reason it is not already done this way?
This issue may apply to configurations other than CMake/GCC, too, but this is the most important one for me.



Discussion:

Resolving patch:
diff --git a/conans/client/generators/cmake.py b/conans/client/generators/cmake.py
--- a/conans/client/generators/cmake.py
+++ b/conans/client/generators/cmake.py
@@ -82,7 +82,7 @@ def _aux_cmake_test_setup(self):
 endmacro()
 
 macro(CONAN_FLAGS_SETUP)
-    include_directories(${CONAN_INCLUDE_DIRS})
+    include_directories(SYSTEM ${CONAN_INCLUDE_DIRS})
     link_directories(${CONAN_LIB_DIRS})
     add_definitions(${CONAN_DEFINES})
 


Test patch:
Repository: conan-io/conan

Issue:
command package compulsory args
`conan package MyLib/0.1@user/channel` raises an error:

```
Packaging
Generating the package
Package folder C:\Users\memsharded\.conan\data\Protobuf\2.6.1\memsharded\testing\package
ERROR: conaninfo.txt does not exist inside of your C:\Users\memsharded\.conan\data\Protobuf\2.6.1\memsharded\testing\build folder. Try to re-build it again to solve it.
```

It requires a package_id or --all flag. It is confusing that not providing either, the error is not very informative. Maybe adding a check that either input is defined?



Discussion:

Resolving patch:
diff --git a/conans/client/command.py b/conans/client/command.py
--- a/conans/client/command.py
+++ b/conans/client/command.py
@@ -269,6 +269,9 @@ def package(self, *args):
         except:
             raise ConanException("Invalid conanfile reference. e.g., OpenSSL/1.0.2e@lasote/stable")
 
+        if not args.all and not args.package:
+            raise ConanException("'conan package': Please specify --all or a package ID")
+
         self._manager.package(reference, args.package, args.only_manifest, args.all)
 
     def export(self, *args):
diff --git a/conans/client/manager.py b/conans/client/manager.py
--- a/conans/client/manager.py
+++ b/conans/client/manager.py
@@ -177,7 +177,10 @@ def package(self, reference, package_id, only_manifest, package_all):
         conan_file_path = self._paths.conanfile(reference)
 
         if package_all:
-            packages_dir = self._paths.packages(reference)
+            if only_manifest:
+                packages_dir = self._paths.packages(reference)
+            else:
+                packages_dir = self._paths.builds(reference)
             if not os.path.exists(packages_dir):
                 raise NotFoundException('%s does not exist' % str(reference))
             packages = [PackageReference(reference, packid)


Test patch:
Repository: conan-io/conan

Issue:
Conan detects ARM architecture, but not defined in settings
Initial auto detection of system conf (to load in conan.conf), fails, as it creates old "arm" arch, while "armv7", etc is what is defined.



Discussion:

Resolving patch:
diff --git a/conans/client/detect.py b/conans/client/detect.py
--- a/conans/client/detect.py
+++ b/conans/client/detect.py
@@ -136,7 +136,15 @@ def detect_defaults_settings(output):
     systems = {'Darwin': 'Macos'}
     result.append(("os", systems.get(platform.system(), platform.system())))
     arch = architectures.get(platform.machine().lower(), platform.machine().lower())
-    arch = 'arm' if arch.startswith('arm') else arch
+    if arch.startswith('arm'):
+        for a in ("armv6", "armv7hf", "armv7", "armv8"):
+            if arch.startswith(a):
+                arch = a
+                break
+        else:
+            output.error("Your ARM '%s' architecture is probably not defined in settings.yml\n"
+                        "Please check your conan.conf and settings.yml files" % arch)
+
     result.append(("arch", arch))
 
     try:


Test patch: