nebius/SWE-rebench-V2-PRs huggingface dataset:
Repository: google/turbinia
Pull request #1164 (2022-09-28 00:48:10)

Check for stderr/out files in Yara task

Fryyyyy: LGTM but need the tests to pass :)

Patch:
diff --git a/turbinia/workers/analysis/yara.py b/turbinia/workers/analysis/yara.py
index 1a82aed..6cf336a 100644
--- a/turbinia/workers/analysis/yara.py
+++ b/turbinia/workers/analysis/yara.py
@@ -104,9 +104,11 @@ class YaraAnalysisTask(TurbiniaTask):
         cmd, result, stderr_file=stderr_file, stdout_file=stdout_file)
 
     if ret != 0:
-      error = "Unknown"
-      with open(stderr_file, 'r') as f:
-        error = f.readlines()
+      if os.path.exists(stderr_file):
+        with open(stderr_file, 'r') as f:
+          error = f.readlines()
+      else:
+        error = "Unknown (no stderr)"
       raise TurbiniaException(
           'Return code: {0:d}. Error: {1!s}'.format(ret, error))
 
@@ -122,7 +124,8 @@ class YaraAnalysisTask(TurbiniaTask):
       with open(stdout_file, 'r') as fraken_report:
         try:
           fraken_output = json.load(fraken_report)
-        except (TypeError, ValueError, json.JSONDecodeError) as exception:
+        except (TypeError, ValueError, FileNotFoundError,
+                json.JSONDecodeError) as exception:
           raise TurbiniaException(
               'Error decoding JSON output from fraken: {0!s}'.format(exception))
         for row in fraken_output:


Test patch:
diff --git a/turbinia/workers/analysis/yara_test.py b/turbinia/workers/analysis/yara_test.py
index c7569d3..9cf09b2 100644
--- a/turbinia/workers/analysis/yara_test.py
+++ b/turbinia/workers/analysis/yara_test.py
@@ -16,10 +16,12 @@
 
 import logging
 import os
+import mock
 import sys
 import tempfile
 import unittest
 
+from turbinia import TurbiniaException
 from turbinia.workers.analysis import yara
 from turbinia.workers.workers_test import TestTurbiniaTaskBase
 
@@ -64,6 +66,16 @@ class YaraAnalysisTaskTest(TestTurbiniaTaskBase):
     self.assertIn('Hadoop', report)
     self.assertIn('Gitlab', report)
 
+  def test_yara_no_stderr(self):
+    """Tests the runFraken method errors with no stderr output."""
+    self.task.execute = mock.MagicMock()
+    # Mocking execute means the stderr file will never get created.
+    self.task.execute.return_value = (1, mock.MagicMock())
+
+    self.assertRaisesRegex(
+        TurbiniaException, '.*Unknown \(no stderr\).*', self.task.runFraken,
+        self.result, self.evidence)
+
 
 if __name__ == '__main__':
   unittest.main()
Repository: google/yapf
Pull request #87 (2015-04-11 04:09:05)

Honour disable/enable comments when --lines is set
Currently, when `--lines` is set, yapf ignores the enable/disable comments.

Is this the intended behaviour?

If not, this pull requests adds a testcase and fixes it.


Patch:
diff --git a/yapf/yapflib/yapf_api.py b/yapf/yapflib/yapf_api.py
index 1e7cce2..9a8ef74 100644
--- a/yapf/yapflib/yapf_api.py
+++ b/yapf/yapflib/yapf_api.py
@@ -182,7 +182,6 @@ def _MarkLinesToFormat(uwlines, lines):
           break
         if uwline.lineno >= start:
           uwline.disable = False
-    return
 
   index = 0
   while index < len(uwlines):


Test patch:
diff --git a/yapftests/yapf_test.py b/yapftests/yapf_test.py
index 3a323c4..bbb605a 100644
--- a/yapftests/yapf_test.py
+++ b/yapftests/yapf_test.py
@@ -561,6 +561,40 @@ class CommandLineTest(unittest.TestCase):
     self.assertIsNone(stderrdata)
     self.assertEqual(reformatted_code.decode('utf-8'), unformatted_code)
 
+  def testDisableWhenSpecifyingLines(self):
+    unformatted_code = textwrap.dedent(u"""\
+        # yapf: disable
+        A = set([
+            'hello',
+            'world',
+        ])
+        # yapf: enable
+        B = set([
+            'hello',
+            'world',
+        ])  # yapf: disable
+        """)
+    expected_formatted_code = textwrap.dedent(u"""\
+        # yapf: disable
+        A = set([
+            'hello',
+            'world',
+        ])
+        # yapf: enable
+        B = set([
+            'hello',
+            'world',
+        ])  # yapf: disable
+        """)
+
+    p = subprocess.Popen(YAPF_BINARY + ['--lines', '1-10'],
+                         stdout=subprocess.PIPE,
+                         stdin=subprocess.PIPE,
+                         stderr=subprocess.STDOUT)
+    reformatted_code, stderrdata = p.communicate(
+        unformatted_code.encode('utf-8'))
+    self.assertIsNone(stderrdata)
+    self.assertEqual(reformatted_code.decode('utf-8'), expected_formatted_code)
 
 if __name__ == '__main__':
   unittest.main()
Repository: google/yapf
Pull request #262 (2016-05-26 17:49:16)

yapf can parse nonlocal as an identifier


Patch:
diff --git a/yapf/yapflib/pytree_utils.py b/yapf/yapflib/pytree_utils.py
index 8bbc44f..be8abf5 100644
--- a/yapf/yapflib/pytree_utils.py
+++ b/yapf/yapflib/pytree_utils.py
@@ -76,6 +76,7 @@ _GRAMMAR_FOR_PY3 = pygram.python_grammar_no_print_statement.copy()
 del _GRAMMAR_FOR_PY3.keywords['exec']
 
 _GRAMMAR_FOR_PY2 = pygram.python_grammar.copy()
+del _GRAMMAR_FOR_PY2.keywords['nonlocal']
 
 
 def ParseCodeToTree(code):


Test patch:
diff --git a/yapftests/pytree_utils_test.py b/yapftests/pytree_utils_test.py
index 8b77f6a..6c650c0 100644
--- a/yapftests/pytree_utils_test.py
+++ b/yapftests/pytree_utils_test.py
@@ -68,6 +68,12 @@ class ParseCodeToTreeTest(unittest.TestCase):
     self.assertEqual(2, len(tree.children))
     self.assertEqual('simple_stmt', pytree_utils.NodeName(tree.children[0]))
 
+  def testClassNotLocal(self):
+    tree = pytree_utils.ParseCodeToTree('class nonlocal: pass\n')
+    self.assertEqual('file_input', pytree_utils.NodeName(tree))
+    self.assertEqual(2, len(tree.children))
+    self.assertEqual('classdef', pytree_utils.NodeName(tree.children[0]))
+
 
 class InsertNodesBeforeAfterTest(unittest.TestCase):
Repository: google/yapf
Pull request #294 (2016-08-19 13:43:13)

Fixing #280 bug
Fix TODO on style module and #280 bug.


Patch:
diff --git a/yapf/yapflib/style.py b/yapf/yapflib/style.py
index ae7333c..5642021 100644
--- a/yapf/yapflib/style.py
+++ b/yapf/yapflib/style.py
@@ -313,7 +313,16 @@ def CreateStyleFromConfig(style_config):
   Raises:
     StyleConfigError: if an unknown style option was encountered.
   """
+  styles = (CreatePEP8Style(), CreateGoogleStyle(),
+             CreateFacebookStyle(), CreateChromiumStyle())
+  def_style = False
   if style_config is None:
+    for style in styles:
+      if _style == style:
+        def_style = True
+        break
+    if not def_style:
+        return _style
     return DEFAULT_STYLE_FACTORY()
   style_factory = _STYLE_NAME_TO_FACTORY.get(style_config.lower())
   if style_factory is not None:
@@ -423,5 +432,5 @@ SETUP_CONFIG = 'setup.cfg'
 # TODO(eliben): For now we're preserving the global presence of a style dict.
 # Refactor this so that the style is passed around through yapf rather than
 # being global.
-_style = {}
+_style = None
 SetGlobalStyle(DEFAULT_STYLE_FACTORY())


Test patch:
diff --git a/yapftests/yapf_test.py b/yapftests/yapf_test.py
index 449a4b5..40a9d3a 100644
--- a/yapftests/yapf_test.py
+++ b/yapftests/yapf_test.py
@@ -26,6 +26,7 @@ import unittest
 
 from yapf.yapflib import py3compat
 from yapf.yapflib import yapf_api
+from yapf.yapflib import style
 
 ROOT_DIR = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
 
@@ -1117,5 +1118,30 @@ class BadInputTest(unittest.TestCase):
     self.assertRaises(SyntaxError, yapf_api.FormatCode, code)
 
 
+class DiffIndentTest(unittest.TestCase):
+
+  @staticmethod
+  def own_style():
+    my_style = style.CreatePEP8Style()
+    my_style['INDENT_WIDTH'] = 3
+    my_style['CONTINUATION_INDENT_WIDTH'] = 3
+    return my_style
+
+  def _Check(self, unformatted_code, expected_formatted_code):
+    formatted_code, _ = yapf_api.FormatCode(
+        unformatted_code, style_config=style.SetGlobalStyle(self.own_style()))
+    self.assertEqual(expected_formatted_code, formatted_code)
+
+  def testSimple(self):
+    unformatted_code = textwrap.dedent(u"""\
+        for i in range(5):
+         print('bar')
+         """)
+    expected_formatted_code = textwrap.dedent(u"""\
+        for i in range(5):
+           print('bar')
+           """)
+    self._Check(unformatted_code, expected_formatted_code)
+
 if __name__ == '__main__':
   unittest.main()
Repository: google/yapf
Pull request #639 (2018-11-05 10:38:15)

Allow custom default for GetDefaultStyleForDir()
Useful if you store a global (e.g. company-wide) fallback style in a file.


Patch:
diff --git a/yapf/yapflib/file_resources.py b/yapf/yapflib/file_resources.py
index 18e6c6b..8317cfc 100644
--- a/yapf/yapflib/file_resources.py
+++ b/yapf/yapflib/file_resources.py
@@ -62,16 +62,18 @@ def GetExcludePatternsForDir(dirname):
   return _GetExcludePatternsFromFile(ignore_file)
 
 
-def GetDefaultStyleForDir(dirname):
+def GetDefaultStyleForDir(dirname, default_style=style.DEFAULT_STYLE):
   """Return default style name for a given directory.
 
   Looks for .style.yapf or setup.cfg in the parent directories.
 
   Arguments:
     dirname: (unicode) The name of the directory.
+    default_style: The style to return if nothing is found. Defaults to the
+                   global default style ('pep8') unless otherwise specified.
 
   Returns:
-    The filename if found, otherwise return the global default (pep8).
+    The filename if found, otherwise return the default style.
   """
   dirname = os.path.abspath(dirname)
   while True:
@@ -98,7 +100,7 @@ def GetDefaultStyleForDir(dirname):
   if os.path.exists(global_file):
     return global_file
 
-  return style.DEFAULT_STYLE
+  return default_style
 
 
 def GetCommandLineFiles(command_line_file_list, recursive, exclude):


Test patch:
diff --git a/yapftests/file_resources_test.py b/yapftests/file_resources_test.py
index 0f413f8..838eaf5 100644
--- a/yapftests/file_resources_test.py
+++ b/yapftests/file_resources_test.py
@@ -73,6 +73,12 @@ class GetDefaultStyleForDirTest(unittest.TestCase):
     style_name = file_resources.GetDefaultStyleForDir(test_file)
     self.assertEqual(style_name, 'pep8')
 
+  def test_no_local_style_custom_default(self):
+    test_file = os.path.join(self.test_tmpdir, 'file.py')
+    style_name = file_resources.GetDefaultStyleForDir(
+        test_file, default_style='custom-default')
+    self.assertEqual(style_name, 'custom-default')
+
   def test_with_local_style(self):
     # Create an empty .style.yapf file in test_tmpdir
     style_file = os.path.join(self.test_tmpdir, '.style.yapf')
Repository: google/yapf
Pull request #1026 (2022-09-23 00:32:49)

Also support Black's `# fmt: off/on` pragma.

gwelymernans: Thanks! Could you add a note in the CHANGELOG?
yilei: Done

Patch:
diff --git a/CHANGELOG b/CHANGELOG
index 5ac3e63..8b29032 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -5,6 +5,7 @@
 ## [0.40.0] UNRELEASED
 ### Added
 - Add a new Python parser to generate logical lines.
+- Added support for `# fmt: on` and `# fmt: off` pragmas.
 ### Changes
 - Moved 'pytree' parsing tools into its own subdirectory.
 - Add support for Python 3.10.
diff --git a/yapf/yapflib/yapf_api.py b/yapf/yapflib/yapf_api.py
index c174514..475d018 100644
--- a/yapf/yapflib/yapf_api.py
+++ b/yapf/yapflib/yapf_api.py
@@ -256,8 +256,8 @@ def _SplitSemicolons(lines):
   return res
 
 
-DISABLE_PATTERN = r'^#.*\byapf:\s*disable\b'
-ENABLE_PATTERN = r'^#.*\byapf:\s*enable\b'
+DISABLE_PATTERN = r'^#.*\b(?:yapf:\s*disable|fmt: ?off)\b'
+ENABLE_PATTERN = r'^#.*\b(?:yapf:\s*enable|fmt: ?on)\b'
 
 
 def _LineRangesToSet(line_ranges):


Test patch:
diff --git a/yapftests/yapf_test.py b/yapftests/yapf_test.py
index 2330f4e..6749bf0 100644
--- a/yapftests/yapf_test.py
+++ b/yapftests/yapf_test.py
@@ -152,6 +152,29 @@ class FormatFileTest(unittest.TestCase):
       formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8')
       self.assertCodeEqual(expected_formatted_code, formatted_code)
 
+  def testFmtOnOff(self):
+    unformatted_code = textwrap.dedent(u"""\
+        if a:    b
+
+        # fmt: off
+        if f:    g
+        # fmt: on
+
+        if h:    i
+        """)
+    expected_formatted_code = textwrap.dedent(u"""\
+        if a: b
+
+        # fmt: off
+        if f:    g
+        # fmt: on
+
+        if h: i
+        """)
+    with utils.TempFileContents(self.test_tmpdir, unformatted_code) as filepath:
+      formatted_code, _, _ = yapf_api.FormatFile(filepath, style_config='pep8')
+      self.assertCodeEqual(expected_formatted_code, formatted_code)
+
   def testDisablePartOfMultilineComment(self):
     unformatted_code = textwrap.dedent(u"""\
         if a:    b
Repository: google/yapf
Pull request #1142 (2023-09-14 07:51:03)

Fix SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED for one-item argument lists
Turn
```
r = f0(1,)
```
into
```
r = f0(
    1,
)
```
in case `SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED` is enabled

alexey-pelykh: @bwendling done

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 73911fb..9beed29 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,7 +8,9 @@
   by removing the "verify" parameter.
 - Changed FORCE_MULTILINE_DICT to override SPLIT_ALL_TOP_LEVEL_COMMA_SEPARATED_VALUES.
 - Adopt pyproject.toml (PEP 517) for build system
+### Fixed
 - Do not treat variables named `match` as the match keyword
+- Fix SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED for one-item argument lists.
 
 ## [0.40.1] 2023-06-20
 ### Fixed
diff --git a/yapf/yapflib/reformatter.py b/yapf/yapflib/reformatter.py
index 319fc9c..ff09525 100644
--- a/yapf/yapflib/reformatter.py
+++ b/yapf/yapflib/reformatter.py
@@ -245,8 +245,13 @@ def _CanPlaceOnSingleLine(line):
   Returns:
     True if the line can or should be added to a single line. False otherwise.
   """
-  token_names = [x.name for x in line.tokens]
-  if (style.Get('FORCE_MULTILINE_DICT') and 'LBRACE' in token_names):
+  token_types = [x.type for x in line.tokens]
+  if (style.Get('SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED') and
+      any(token_types[token_index - 1] == token.COMMA
+          for token_index, token_type in enumerate(token_types[1:], start=1)
+          if token_type == token.RPAR)):
+    return False
+  if (style.Get('FORCE_MULTILINE_DICT') and token.LBRACE in token_types):
     return False
   indent_amt = style.Get('INDENT_WIDTH') * line.depth
   last = line.last


Test patch:
diff --git a/yapftests/reformatter_basic_test.py b/yapftests/reformatter_basic_test.py
index aae46d6..c2200a9 100644
--- a/yapftests/reformatter_basic_test.py
+++ b/yapftests/reformatter_basic_test.py
@@ -2275,6 +2275,8 @@ xxxxxxxxxxx, yyyyyyyyyyyy, vvvvvvvvv)
         a_very_long_function_name(long_argument_name_1, long_argument_name_2, long_argument_name_3, long_argument_name_4,)
 
         r =f0 (1,  2,3,)
+
+        r =f0 (1,)
     """)  # noqa
     expected_formatted_code = textwrap.dedent("""\
         function_name(argument_name_1=1, argument_name_2=2, argument_name_3=3)
@@ -2303,6 +2305,10 @@ xxxxxxxxxxx, yyyyyyyyyyyy, vvvvvvvvv)
             2,
             3,
         )
+
+        r = f0(
+            1,
+        )
     """)
 
     try:
Repository: google/turbinia
Pull request #1231 (2023-01-12 00:08:15)

Fix for duplicate UUID issue in XFS image mounts
Fixes issue where two workers on same machine try to mount an XFS image causing an error with msg`Filesystem has duplicate UUID`, which has been the culprit to some failed Tasks we've been seeing from testing.

Adding in `-o nouuid` flag to ignore the UUID for XFS images when mounting, which in our case is fine as the UUID is not being used for anything in Turbinia.

jleaniz: Fxies #1234

Patch:
diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile
index b6ed944..f40538c 100644
--- a/docker/worker/Dockerfile
+++ b/docker/worker/Dockerfile
@@ -87,8 +87,8 @@ COPY turbinia/config/rules/*.yar /opt/signature-base/yara/
 
 RUN mkdir -p /opt/fraken/yara && chown -R turbinia:turbinia /opt/fraken
 COPY --chown=turbinia:turbinia tools/fraken/* /opt/fraken/
-RUN cd /opt/fraken/yara && curl -s https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p' | xargs -n1 wget -O - -q | tar -xz --strip-components=1 \
-    && ./bootstrap.sh \
+RUN cd /opt/fraken/yara && curl -s -L --retry 5 "$(curl -s -L --retry 5 https://api.github.com/repos/VirusTotal/Yara/releases/latest | sed -n 's/.*"tarball_url": "\(.*\)",.*/\1/p')"  | tar -xz --strip-components=1
+RUN cd /opt/fraken/yara && ./bootstrap.sh \
     && ./configure \
     && make && sudo make install && sudo ldconfig
 RUN cd /opt/fraken && go build -ldflags="-linkmode=external -extldflags=-ljemalloc" -o fraken
diff --git a/turbinia/api/cli/requirements.txt b/turbinia/api/cli/requirements.txt
index b03cb4c..dca9a90 100644
--- a/turbinia/api/cli/requirements.txt
+++ b/turbinia/api/cli/requirements.txt
@@ -1,2 +1,1 @@
-turbinia-api-client
 click
diff --git a/turbinia/processors/mount_local.py b/turbinia/processors/mount_local.py
index 7b77937..9b4b7d3 100644
--- a/turbinia/processors/mount_local.py
+++ b/turbinia/processors/mount_local.py
@@ -495,7 +495,7 @@ def PreprocessMountPartition(partition_path, filesystem_type):
     # everything read-only.
     mount_cmd.extend(['-o', 'noload'])
   elif filesystem_type == 'XFS':
-    mount_cmd.extend(['-o', 'norecovery'])
+    mount_cmd.extend(['-o', 'norecovery', '-o', 'nouuid'])
   mount_cmd.extend([partition_path, mount_path])
 
   log.info('Running: {0:s}'.format(' '.join(mount_cmd)))


Test patch:
diff --git a/turbinia/processors/mount_local_test.py b/turbinia/processors/mount_local_test.py
index 6790e19..6f2e9b4 100644
--- a/turbinia/processors/mount_local_test.py
+++ b/turbinia/processors/mount_local_test.py
@@ -371,8 +371,8 @@ class MountLocalProcessorTest(unittest.TestCase):
     mock_subprocess.reset_mock()
     mount_path = mount_local.PreprocessMountPartition('/dev/loop0', 'XFS')
     expected_args = [
-        'sudo', 'mount', '-o', 'ro', '-o', 'norecovery', '/dev/loop0',
-        '/mnt/turbinia/turbinia0ckdntz0'
+        'sudo', 'mount', '-o', 'ro', '-o', 'norecovery', '-o', 'nouuid',
+        '/dev/loop0', '/mnt/turbinia/turbinia0ckdntz0'
     ]
     mock_subprocess.assert_called_once_with(expected_args)
     self.assertEqual(mount_path, '/mnt/turbinia/turbinia0ckdntz0')