ronantakizawa/codereview-bench huggingface dataset:
Repository: projectdiscovery/katana
File: internal/runner/executer.go (Go)

Code before:
package runner

import (
	"strings"

	"github.com/projectdiscovery/gologger"
	"github.com/projectdiscovery/katana/pkg/utils"
	errorutil "github.com/projectdiscovery/utils/errors"
	urlutil "github.com/projectdiscovery/utils/url"
	"github.com/remeh/sizedwaitgroup"
)

// ExecuteCrawling executes the crawling main loop
func (r *Runner) ExecuteCrawling() error {
	if r.crawler == nil {
		return errorutil.New("crawler is not initialized")
	}
	inputs := r.parseInputs()
	if len(inputs) == 0 {
		return errorutil.New("no input provided for crawling")
	}

	if r.options.PathClimb {
		for _, input := range inputs {
			extractedParentURLs := utils.ExtractParentPaths(input)
			inputs = append(inputs, extractedParentURLs...)
		}
	}

	for _, input := range inputs {
		_ = r.state.InFlightUrls.Set(addSchemeIfNotExists(input), struct{}{})
	}

	defer func() {
		if err := r.crawler.Close(); err != nil {
			gologger.Error().Msgf("Error closing crawler: %v\n", err)
		}
	}()

	wg := sizedwaitgroup.New(r.options.Parallelism)
	for _, input := range inputs {
		if !r.networkpolicy.Validate(input) {
			gologger.Info().Msgf("Skipping excluded host %s", input)
			continue
		}
		wg.Add()
		input = addSchemeIfNotExists(input)
		go func(input string) {

Review comment (refactor):
I think this might create redundant crawl sessions. Anyway the same URLs should be visited within the Do(..) method later on as navigation requests are passing by. I think we can safely remove this. What do you think?

Code after:
package runner

import (
	"strings"

	"github.com/projectdiscovery/gologger"
	errorutil "github.com/projectdiscovery/utils/errors"
	urlutil "github.com/projectdiscovery/utils/url"
	"github.com/remeh/sizedwaitgroup"
)

// ExecuteCrawling executes the crawling main loop
func (r *Runner) ExecuteCrawling() error {
	if r.crawler == nil {
		return errorutil.New("crawler is not initialized")
	}
	inputs := r.parseInputs()
	if len(inputs) == 0 {
		return errorutil.New("no input provided for crawling")
	}

	for _, input := range inputs {
		_ = r.state.InFlightUrls.Set(addSchemeIfNotExists(input), struct{}{})
	}

	defer func() {
		if err := r.crawler.Close(); err != nil {
			gologger.Error().Msgf("Error closing crawler: %v\n", err)
		}
	}()

	wg := sizedwaitgroup.New(r.options.Parallelism)
	for _, input := range inputs {
		if !r.networkpolicy.Validate(input) {
			gologger.Info().Msgf("Skipping excluded host %s", input)
			continue
		}
		wg.Add()
		input = addSchemeIfNotExists(input)
		go func(input string) {
			defer wg.Done()

			if err := r.crawler.Crawl(input); err != nil {
				gologger.Warning().Msgf("Could not crawl %s: %s", input, err)
			}
			r.state.InFlightUrls.Delete(input)
		}(input)
	}
Repository: psf/requests
File: setup.py (Python)

Code before:
            from multiprocessing import cpu_count
            self.pytest_args = ['-n', str(cpu_count()), '--boxed']
        except (ImportError, NotImplementedError):
            self.pytest_args = ['-n', '1', '--boxed']

    def finalize_options(self):
        TestCommand.finalize_options(self)
        self.test_args = []
        self.test_suite = True

    def run_tests(self):
        import pytest

        errno = pytest.main(self.pytest_args)
        sys.exit(errno)

# 'setup.py publish' shortcut.
if sys.argv[-1] == 'publish':
    os.system('python setup.py sdist bdist_wheel')
    os.system('twine upload dist/*')
    sys.exit()

packages = ['requests']

requires = [
    'charset_normalizer>=1.3.9,<2; python_version >= "3"',
    'chardet>=3.0.2,<5; python_version < "3"',
    'idna>=2.5,<3',
    'urllib3>=1.21.1,<1.27',
    'certifi>=2017.4.17'

]
test_requirements = [
    'pytest-httpbin==0.0.7',
    'pytest-cov',
    'pytest-mock',
    'pytest-xdist',
    'PySocks>=1.5.6, !=1.5.7',
    'pytest>=3'
]

about = {}
with open(os.path.join(here, 'requests', '__version__.py'), 'r', 'utf-8') as f:
    exec(f.read(), about)

with open('README.md', 'r', 'utf-8') as f:
    readme = f.read()

setup(
    name=about['__title__'],
    version=about['__version__'],

Review comment (suggestion):
```suggestion
    'charset_normalizer~=1.4.0,<2; python_version >= "3"',
```

Code after:
            from multiprocessing import cpu_count
            self.pytest_args = ['-n', str(cpu_count()), '--boxed']
        except (ImportError, NotImplementedError):
            self.pytest_args = ['-n', '1', '--boxed']

    def finalize_options(self):
        TestCommand.finalize_options(self)
        self.test_args = []
        self.test_suite = True

    def run_tests(self):
        import pytest

        errno = pytest.main(self.pytest_args)
        sys.exit(errno)

# 'setup.py publish' shortcut.
if sys.argv[-1] == 'publish':
    os.system('python setup.py sdist bdist_wheel')
    os.system('twine upload dist/*')
    sys.exit()

packages = ['requests']

requires = [
    'charset_normalizer~=2.0.0; python_version >= "3"',
    'chardet>=3.0.2,<5; python_version < "3"',
    'idna>=2.5,<3',
    'urllib3>=1.21.1,<1.27',
    'certifi>=2017.4.17'

]
test_requirements = [
    'pytest-httpbin==0.0.7',
    'pytest-cov',
    'pytest-mock',
    'pytest-xdist',
    'PySocks>=1.5.6, !=1.5.7',
    'pytest>=3'
]

about = {}
with open(os.path.join(here, 'requests', '__version__.py'), 'r', 'utf-8') as f:
    exec(f.read(), about)

with open('README.md', 'r', 'utf-8') as f:
    readme = f.read()

setup(
    name=about['__title__'],
    version=about['__version__'],
Repository: psf/requests
File: setup.py (Python)

Code before:
            from multiprocessing import cpu_count
            self.pytest_args = ['-n', str(cpu_count()), '--boxed']
        except (ImportError, NotImplementedError):
            self.pytest_args = ['-n', '1', '--boxed']

    def finalize_options(self):
        TestCommand.finalize_options(self)
        self.test_args = []
        self.test_suite = True

    def run_tests(self):
        import pytest

        errno = pytest.main(self.pytest_args)
        sys.exit(errno)

# 'setup.py publish' shortcut.
if sys.argv[-1] == 'publish':
    os.system('python setup.py sdist bdist_wheel')
    os.system('twine upload dist/*')
    sys.exit()

packages = ['requests']

requires = [
    'charset_normalizer~=1.4.0,<2; python_version >= "3"',
    'chardet>=3.0.2,<5; python_version < "3"',
    'idna>=2.5,<3',
    'urllib3>=1.21.1,<1.27',
    'certifi>=2017.4.17'

]
test_requirements = [
    'pytest-httpbin==0.0.7',
    'pytest-cov',
    'pytest-mock',
    'pytest-xdist',
    'PySocks>=1.5.6, !=1.5.7',
    'pytest>=3'
]

about = {}
with open(os.path.join(here, 'requests', '__version__.py'), 'r', 'utf-8') as f:
    exec(f.read(), about)

with open('README.md', 'r', 'utf-8') as f:
    readme = f.read()

setup(
    name=about['__title__'],
    version=about['__version__'],

Review comment (suggestion):
```suggestion
    'charset_normalizer~=1.4.0; python_version >= "3"',
```

Code after:
            from multiprocessing import cpu_count
            self.pytest_args = ['-n', str(cpu_count()), '--boxed']
        except (ImportError, NotImplementedError):
            self.pytest_args = ['-n', '1', '--boxed']

    def finalize_options(self):
        TestCommand.finalize_options(self)
        self.test_args = []
        self.test_suite = True

    def run_tests(self):
        import pytest

        errno = pytest.main(self.pytest_args)
        sys.exit(errno)

# 'setup.py publish' shortcut.
if sys.argv[-1] == 'publish':
    os.system('python setup.py sdist bdist_wheel')
    os.system('twine upload dist/*')
    sys.exit()

packages = ['requests']

requires = [
    'charset_normalizer~=2.0.0; python_version >= "3"',
    'chardet>=3.0.2,<5; python_version < "3"',
    'idna>=2.5,<3',
    'urllib3>=1.21.1,<1.27',
    'certifi>=2017.4.17'

]
test_requirements = [
    'pytest-httpbin==0.0.7',
    'pytest-cov',
    'pytest-mock',
    'pytest-xdist',
    'PySocks>=1.5.6, !=1.5.7',
    'pytest>=3'
]

about = {}
with open(os.path.join(here, 'requests', '__version__.py'), 'r', 'utf-8') as f:
    exec(f.read(), about)

with open('README.md', 'r', 'utf-8') as f:
    readme = f.read()

setup(
    name=about['__title__'],
    version=about['__version__'],
Repository: psf/requests
File: setup.py (Python)

Code before:
    license=about['__license__'],
    zip_safe=False,
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'Natural Language :: English',
        'License :: OSI Approved :: Apache Software License',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: 3.7',
        'Programming Language :: Python :: 3.8',
        'Programming Language :: Python :: 3.9',
        'Programming Language :: Python :: Implementation :: CPython',
        'Programming Language :: Python :: Implementation :: PyPy'
    ],
    cmdclass={'test': PyTest},
    tests_require=test_requirements,
    extras_require={
        'security': ['pyOpenSSL >= 0.14', 'cryptography>=1.3.4'],
        'socks': ['PySocks>=1.5.6, !=1.5.7'],
        'socks:sys_platform == "win32" and python_version == "2.7"': ['win_inet_pton'],
        'lgpl': ['chardet>=3.0.2,<5']
    },
    project_urls={
        'Documentation': 'https://requests.readthedocs.io',
        'Source': 'https://github.com/psf/requests',
    },
)


Review comment (suggestion):
This seems somewhat disingenuous. For one thing, it's not the _only_ way to get `lgpl` dependencies. For another, it really exists to just force `chardet` to be installed on py3 so should be named something more like `use_chardet_on_py3`

Code after:
    license=about['__license__'],
    zip_safe=False,
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'Natural Language :: English',
        'License :: OSI Approved :: Apache Software License',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: 3.7',
        'Programming Language :: Python :: 3.8',
        'Programming Language :: Python :: 3.9',
        'Programming Language :: Python :: Implementation :: CPython',
        'Programming Language :: Python :: Implementation :: PyPy'
    ],
    cmdclass={'test': PyTest},
    tests_require=test_requirements,
    extras_require={
        'security': ['pyOpenSSL >= 0.14', 'cryptography>=1.3.4'],
        'socks': ['PySocks>=1.5.6, !=1.5.7'],
        'socks:sys_platform == "win32" and python_version == "2.7"': ['win_inet_pton'],
        'use_chardet_on_py3': ['chardet>=3.0.2,<5']
    },
    project_urls={
        'Documentation': 'https://requests.readthedocs.io',
        'Source': 'https://github.com/psf/requests',
    },
)
Repository: psf/requests
File: requests/exceptions.py (Python)

Code before:
except ImportError:  # Python 2, ValueError is raised
    StandardJSONDecodeError = ValueError

try:
    from simplejson import JSONDecodeError as SimpleJSONDecodeError
except ImportError:
    SimpleJSONDecodeError = Exception


class RequestException(IOError):
    """There was an ambiguous exception that occurred while handling your
    request.
    """

    def __init__(self, *args, **kwargs):
        """Initialize RequestException with `request` and `response` objects."""
        response = kwargs.pop('response', None)
        self.response = response
        self.request = kwargs.pop('request', None)
        if (response is not None and not self.request and
                hasattr(response, 'request')):
            self.request = self.response.request
        super(RequestException, self).__init__(*args, **kwargs)


class JSONDecodeError(StandardJSONDecodeError, SimpleJSONDecodeError):
    """Couldn't decode the text into json"""


class InvalidJSONError(RequestException):
    """A JSON error occurred."""


class HTTPError(RequestException):
    """An HTTP error occurred."""


class ConnectionError(RequestException):
    """A Connection error occurred."""


class ProxyError(ConnectionError):
    """A proxy error occurred."""


class SSLError(ConnectionError):
    """An SSL error occurred."""


class Timeout(RequestException):
    """The request timed out.

Review comment (suggestion):
Our custom exceptions in Requests should ideally fall under a catch-all parent exception, `RequestException`. That allows users to catch everything thrown by Requests.

Code after:

class RequestException(IOError):
    """There was an ambiguous exception that occurred while handling your
    request.
    """

    def __init__(self, *args, **kwargs):
        """Initialize RequestException with `request` and `response` objects."""
        response = kwargs.pop('response', None)
        self.response = response
        self.request = kwargs.pop('request', None)
        if (response is not None and not self.request and
                hasattr(response, 'request')):
            self.request = self.response.request
        super(RequestException, self).__init__(*args, **kwargs)


class InvalidJSONError(RequestException):
    """A JSON error occurred."""


class JSONDecodeError(InvalidJSONError, CompatJSONDecodeError):
    """Couldn't decode the text into json"""


class HTTPError(RequestException):
    """An HTTP error occurred."""


class ConnectionError(RequestException):
    """A Connection error occurred."""


class ProxyError(ConnectionError):
    """A proxy error occurred."""


class SSLError(ConnectionError):
    """An SSL error occurred."""


class Timeout(RequestException):
    """The request timed out.

    Catching this error will catch both
    :exc:`~requests.exceptions.ConnectTimeout` and
    :exc:`~requests.exceptions.ReadTimeout` errors.
    """


class ConnectTimeout(ConnectionError, Timeout):
Repository: psf/requests
File: tests/test_requests.py (Python)

Code before:
            ),
        )
    )
    def test_parameters_for_nonstandard_schemes(self, input, params, expected):
        """
        Setting parameters for nonstandard schemes is allowed if those schemes
        begin with "http", and is forbidden otherwise.
        """
        r = requests.Request('GET', url=input, params=params)
        p = r.prepare()
        assert p.url == expected

    def test_post_json_nan(self, httpbin):
        data = {"foo": float("nan")}
        with pytest.raises(requests.exceptions.InvalidJSONError):
          r = requests.post(httpbin('post'), json=data)

    def test_json_decode_compatibility(self, httpbin):
        r = requests.get(httpbin('bytes/20'))
        data = r.text
        with pytest.raises(requests.exceptions.JSONDecodeError) as excinfo:
            r.json()
        assert isinstance(excinfo.value, RequestException)
        assert isinstance(excinfo.value, JSONDecodeError)
        assert data not in str(excinfo.value)
        if is_py3:
            assert excinfo.value.doc == data


Review comment (suggestion):
Also somewhat of a nit. It may be better to just make this its own test with a `@unittest.skipIf(not is_py3)` at the top.

e.g.

```python
@unittest.skipIf(not is_py3)
def test_json_decode_persists_doc_attr(self, httpbin):
    r = requests.get(httpbin('bytes/20'))
    with pytest.raises(requests.exceptions.JSONDecodeError) as e:
        r.json()
    assert e.doc == r.text
```
        


Code after:
            ),
        )
    )
    def test_parameters_for_nonstandard_schemes(self, input, params, expected):
        """
        Setting parameters for nonstandard schemes is allowed if those schemes
        begin with "http", and is forbidden otherwise.
        """
        r = requests.Request('GET', url=input, params=params)
        p = r.prepare()
        assert p.url == expected

    def test_post_json_nan(self, httpbin):
        data = {"foo": float("nan")}
        with pytest.raises(requests.exceptions.InvalidJSONError):
          r = requests.post(httpbin('post'), json=data)

    def test_json_decode_compatibility(self, httpbin):
        r = requests.get(httpbin('bytes/20'))
        with pytest.raises(requests.exceptions.JSONDecodeError) as excinfo:
            r.json()
        assert isinstance(excinfo.value, RequestException)
        assert isinstance(excinfo.value, JSONDecodeError)
        assert r.text not in str(excinfo.value)

    @pytest.mark.skipif(not is_py3, reason="doc attribute is only present on py3")
    def test_json_decode_persists_doc_attr(self, httpbin):
        r = requests.get(httpbin('bytes/20'))
        with pytest.raises(requests.exceptions.JSONDecodeError) as excinfo:
            r.json()
        assert excinfo.value.doc == r.text
Repository: psf/requests
File: tests/test_requests.py (Python)

Code before:
        """
        Setting parameters for nonstandard schemes is allowed if those schemes
        begin with "http", and is forbidden otherwise.
        """
        r = requests.Request('GET', url=input, params=params)
        p = r.prepare()
        assert p.url == expected

    def test_post_json_nan(self, httpbin):
        data = {"foo": float("nan")}
        with pytest.raises(requests.exceptions.InvalidJSONError):
          r = requests.post(httpbin('post'), json=data)

    def test_json_decode_compatibility(self, httpbin):
        r = requests.get(httpbin('bytes/20'))
        data = r.text
        with pytest.raises(requests.exceptions.JSONDecodeError) as excinfo:
            r.json()
        assert isinstance(excinfo.value, RequestException)
        assert isinstance(excinfo.value, JSONDecodeError)
        assert data not in str(excinfo.value)

    @pytest.mark.skipif(not is_py3, reason="doc attribute is only present on py3")
    def test_json_decode_persists_doc_attr(self, httpbin):
        r = requests.get(httpbin('bytes/20'))
        data = r.text
        with pytest.raises(requests.exceptions.JSONDecodeError) as excinfo:
            r.json()
        assert data not in str(excinfo.value)
        assert excinfo.value.doc == data


Review comment (suggestion):
nit; since we're only using this once we can probably just use r.text in the final assertion as shown in the original proposal.

Code after:
        """
        Setting parameters for nonstandard schemes is allowed if those schemes
        begin with "http", and is forbidden otherwise.
        """
        r = requests.Request('GET', url=input, params=params)
        p = r.prepare()
        assert p.url == expected

    def test_post_json_nan(self, httpbin):
        data = {"foo": float("nan")}
        with pytest.raises(requests.exceptions.InvalidJSONError):
          r = requests.post(httpbin('post'), json=data)

    def test_json_decode_compatibility(self, httpbin):
        r = requests.get(httpbin('bytes/20'))
        with pytest.raises(requests.exceptions.JSONDecodeError) as excinfo:
            r.json()
        assert isinstance(excinfo.value, RequestException)
        assert isinstance(excinfo.value, JSONDecodeError)
        assert r.text not in str(excinfo.value)

    @pytest.mark.skipif(not is_py3, reason="doc attribute is only present on py3")
    def test_json_decode_persists_doc_attr(self, httpbin):
        r = requests.get(httpbin('bytes/20'))
        with pytest.raises(requests.exceptions.JSONDecodeError) as excinfo:
            r.json()
        assert excinfo.value.doc == r.text
Repository: psf/requests
File: tests/compat.py (Python)

Code before:
# -*- coding: utf-8 -*-

try:
    import StringIO
except ImportError:
    import io as StringIO

try:
    from cStringIO import StringIO as cStringIO
except ImportError:
    cStringIO = None

def u(s):
    return s


Review comment (refactor):
Should we go through and remove all uses of `u(...)` in the tests to simplify them?

Code after:
# -*- coding: utf-8 -*-
import warnings

try:
    import StringIO
except ImportError:
    import io as StringIO

try:
    from cStringIO import StringIO as cStringIO
except ImportError:
    cStringIO = None


def u(s):
    warnings.warn(
        (
            "This helper function is no longer relevant in Python 3. "
            "Usage of this alias should be discontinued as it will be "
            "removed in a future release of Requests."
        ),
        DeprecationWarning,
    )
    return s
Repository: psf/requests
File: setup.py (Python)

Code before:
#!/usr/bin/env python
import os
import sys

from codecs import open

from setuptools import setup
from setuptools.command.test import test as TestCommand


CURRENT_PYTHON = sys.version_info[:2]
REQUIRED_PYTHON = (3, 7)

if CURRENT_PYTHON < REQUIRED_PYTHON:
    sys.stderr.write(
        """
==========================
Unsupported Python version
==========================
This version of Requests requires at least Python {}.{}, but
you're trying to install it on Python {}.{}. To resolve this,
consider upgrading to a supported Python version.

If you can't upgrade your Python version, you'll need to
pin to an older version of Requests:
    python -m pip install "requests<2.28"
""".format(
            *(REQUIRED_PYTHON + CURRENT_PYTHON)
        )
    )
    sys.exit(1)

class PyTest(TestCommand):
    user_options = [('pytest-args=', 'a', "Arguments to pass into py.test")]

    def initialize_options(self):
        TestCommand.initialize_options(self)
        try:
            from multiprocessing import cpu_count
            self.pytest_args = ['-n', str(cpu_count()), '--boxed']
        except (ImportError, NotImplementedError):
            self.pytest_args = ['-n', '1', '--boxed']

    def finalize_options(self):
        TestCommand.finalize_options(self)
        self.test_args = []
        self.test_suite = True

    def run_tests(self):
        import pytest


Review comment (suggestion):
```suggestion
pin to an older version of Requests (<2.28).
```

Let's just go all the way here. :P

Code after:
#!/usr/bin/env python
import os
import sys

from codecs import open

from setuptools import setup
from setuptools.command.test import test as TestCommand


CURRENT_PYTHON = sys.version_info[:2]
REQUIRED_PYTHON = (3, 7)

if CURRENT_PYTHON < REQUIRED_PYTHON:
    sys.stderr.write(
        """
==========================
Unsupported Python version
==========================
This version of Requests requires at least Python {}.{}, but
you're trying to install it on Python {}.{}. To resolve this,
consider upgrading to a supported Python version.

If you can't upgrade your Python version, you'll need to
pin to an older version of Requests (<2.28).
""".format(
            *(REQUIRED_PYTHON + CURRENT_PYTHON)
        )
    )
    sys.exit(1)

class PyTest(TestCommand):
    user_options = [('pytest-args=', 'a', "Arguments to pass into py.test")]

    def initialize_options(self):
        TestCommand.initialize_options(self)
        try:
            from multiprocessing import cpu_count
            self.pytest_args = ['-n', str(cpu_count()), '--boxed']
        except (ImportError, NotImplementedError):
            self.pytest_args = ['-n', '1', '--boxed']

    def finalize_options(self):
        TestCommand.finalize_options(self)
        self.test_args = []
        self.test_suite = True

    def run_tests(self):
        import pytest

        errno = pytest.main(self.pytest_args)