Repository: scrapinghub/scrapinghub-entrypoint-scrapy

Issue:
Scrapy 2.6 support
```
Traceback (most recent call last):
  File "/usr/local/bin/shub-image-info", line 8, in <module>
    sys.exit(shub_image_info())
  File "/usr/local/lib/python3.10/site-packages/sh_scrapy/crawl.py", line 209, in shub_image_info
    _run_usercode(None, ['scrapy', 'shub_image_info'] + sys.argv[1:],
  File "/usr/local/lib/python3.10/site-packages/sh_scrapy/crawl.py", line 148, in _run_usercode
    _run(args, settings)
  File "/usr/local/lib/python3.10/site-packages/sh_scrapy/crawl.py", line 103, in _run
    _run_scrapy(args, settings)
  File "/usr/local/lib/python3.10/site-packages/sh_scrapy/crawl.py", line 111, in _run_scrapy
    execute(settings=settings)
  File "/usr/local/lib/python3.10/site-packages/scrapy/cmdline.py", line 140, in execute
    cmd.add_options(parser)
  File "/usr/local/lib/python3.10/site-packages/sh_scrapy/commands/shub_image_info.py", line 23, in add_options
    parser.add_option("--debug", action="store_true",
AttributeError: 'ArgumentParser' object has no attribute 'add_option'. Did you mean: '_add_action'?
```

Caused by https://github.com/scrapy/scrapy/pull/5374

Discussion:
elacuesta: Good catch, thanks!

Patch:
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
new file mode 100644
index 0000000..e0f4d6b
--- /dev/null
+++ b/.github/workflows/publish.yml
@@ -0,0 +1,25 @@
+name: Publish
+on:
+  release:
+    types: [published]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+
+    - name: Publish to PyPI
+      run: |
+        pip install --upgrade pip
+        pip install --upgrade setuptools wheel twine
+        python setup.py sdist bdist_wheel
+        export TWINE_USERNAME=__token__
+        export TWINE_PASSWORD=${{ secrets.PYPI_TOKEN }}
+        twine upload dist/*
diff --git a/.gitignore b/.gitignore
index 39b16d7..5ff28cb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,4 @@ dist
 .eggs/
 .tox
 /.coverage
+coverage.xml
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index ddb4e78..0000000
--- a/.travis.yml
+++ /dev/null
@@ -1,41 +0,0 @@
-language: python
-branches:
-  only:
-  - master
-  - /^\d\.\d+$/
-  - /^v\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
-matrix:
-  include:
-    - python: 2.7
-      env: TOXENV=py27
-    - python: 2.7
-      env: TOXENV=py27-scrapy105
-    - python: 3.5
-      env: TOXENV=py35
-    - python: 3.6
-      env: TOXENV=py36
-    - python: 3.7
-      env: TOXENV=py37
-    - python: 3.8
-      env: TOXENV=py38
-    - python: 3.9
-      env: TOXENV=py39
-install:
-  - pip install tox
-  - pip install -U tox coverage codecov pytest-xdist
-script:
-  - tox
-after_success:
-  - codecov
-cache:
-  directories:
-  - $HOME/.cache/pip
-deploy:
-  provider: pypi
-  distributions: sdist bdist_wheel
-  user: scrapinghub
-  on:
-    tags: true
-    condition: $TOXENV == py36 && $TRAVIS_TAG =~ ^v[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$
-  password:
-    secure: 05NUQdIRHlT+9T85SJXWfg35q3bk7vlpkfHfwisDkbcJnD5+kymRTzbSzhABQxh3bpc0QHRc2bF1PfgCjumaJMarp3K0unsOdptleUwhorizkyvnD8xzxWd+8GFUDBzoCLJqC9TQK05wWrTqbGbZ+stPnatEx7SfJEHbxhKeL7M=
diff --git a/setup.py b/setup.py
index 2a51fba..ed6b134 100644
--- a/setup.py
+++ b/setup.py
@@ -18,6 +18,7 @@ setup(
             'shub-image-info = sh_scrapy.crawl:shub_image_info',
         ],
     },
+    python_requires='>=3.5',
     classifiers=[
         'Framework :: Scrapy',
         'Development Status :: 5 - Production/Stable',
@@ -25,13 +26,12 @@ setup(
         'License :: OSI Approved :: BSD License',
         'Operating System :: OS Independent',
         'Programming Language :: Python',
-        'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+        'Programming Language :: Python :: 3.9',
         'Topic :: Utilities',
     ],
 )
diff --git a/sh_scrapy/commands/shub_image_info.py b/sh_scrapy/commands/shub_image_info.py
index 356691f..844cfb2 100644
--- a/sh_scrapy/commands/shub_image_info.py
+++ b/sh_scrapy/commands/shub_image_info.py
@@ -20,10 +20,17 @@ class Command(ScrapyCommand):
 
     def add_options(self, parser):
         super(Command, self).add_options(parser)
-        parser.add_option("--debug", action="store_true",
-                          help="add debugging information such as "
-                               "list of installed Debian packages "
-                               "and Python packages.")
+        # backward compatibility for optparse/argparse
+        try:
+            add_argument = parser.add_argument
+        except AttributeError:
+            add_argument = parser.add_option
+        add_argument(
+            "--debug",
+            action="store_true",
+            help="add debugging information such as list of "
+                 "installed Debian packages and Python packages.",
+        )
 
     def run(self, args, opts):
         result = {
diff --git a/sh_scrapy/extension.py b/sh_scrapy/extension.py
index 4013a0a..0192bfb 100644
--- a/sh_scrapy/extension.py
+++ b/sh_scrapy/extension.py
@@ -1,11 +1,12 @@
 import logging
+from contextlib import suppress
 from weakref import WeakKeyDictionary
 
+import scrapy
 from scrapy import signals
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.exporters import PythonItemExporter
 from scrapy.http import Request
-from scrapy.item import BaseItem
 from scrapy.utils.deprecate import create_deprecated_class
 from scrapy.utils.request import request_fingerprint
 
@@ -20,8 +21,12 @@ from sh_scrapy.writer import pipe_writer
 try:
     from itemadapter import is_item
 except ImportError:
+    _base_item_cls = [dict, scrapy.item.Item]
+    with suppress(AttributeError):
+        _base_item_cls.append(scrapy.item.BaseItem)
+
     def is_item(item):
-        return isinstance(item, (dict, BaseItem))
+        return isinstance(item, tuple(_base_item_cls))
 
 
 class HubstorageExtension(object):
diff --git a/tox.ini b/tox.ini
index ca63d48..6de1fe4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,6 +1,6 @@
 # tox.ini
 [tox]
-envlist = py27, py27-scrapy105, py35, py36
+envlist = py35-scrapy18, py35, py36, py37, py38, py39, py310
 
 [testenv]
 deps =
@@ -9,6 +9,7 @@ deps =
     mock
     hubstorage
     six
-    py27-scrapy105: Scrapy==1.0.5
+    packaging
+    py35-scrapy18: Scrapy==1.8
 commands =
-    py.test --cov=sh_scrapy --cov-report= {posargs:}
+    pytest --verbose --cov=sh_scrapy --cov-report=term-missing --cov-report=html --cov-report=xml {posargs: sh_scrapy tests}


Test patch:
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 0000000..aae3e6c
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,50 @@
+name: Tests
+
+on: [push, pull_request]
+
+jobs:
+  tests-ubuntu:
+    name: "Test: py${{ matrix.python-version }}, Ubuntu"
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - python-version: "3.5"
+          env:
+            TOXENV: py35-scrapy18
+        - python-version: "3.5"
+          env:
+            TOXENV: py35
+        - python-version: "3.6"
+          env:
+            TOXENV: py36
+        - python-version: "3.7"
+          env:
+            TOXENV: py37
+        - python-version: "3.8"
+          env:
+            TOXENV: py38
+        - python-version: "3.9"
+          env:
+            TOXENV: py39
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Update pip & install tox
+      run: |
+        pip install -U pip
+        pip install tox
+
+    - name: Run tests
+      env: ${{ matrix.env }}
+      run: tox
+
+    - name: Upload coverage report
+      run: bash <(curl -s https://codecov.io/bash)
diff --git a/tests/test_command.py b/tests/test_command.py
new file mode 100644
index 0000000..4040851
--- /dev/null
+++ b/tests/test_command.py
@@ -0,0 +1,38 @@
+
+from argparse import ArgumentParser
+from optparse import OptionParser
+
+import pytest
+import scrapy
+from packaging import version
+
+from sh_scrapy.commands.shub_image_info import Command
+
+
+@pytest.fixture
+def command():
+    command = Command()
+    command.settings = scrapy.settings.Settings()
+    return command
+
+
+@pytest.mark.skipif(
+    version.parse(scrapy.__version__) >= version.parse("2.6"),
+    reason="Scrapy>=2.6 uses argparse"
+)
+def test_optparse(command):
+    parser = OptionParser()
+    command.add_options(parser)
+    options = parser.parse_args(["--debug"])
+    assert options[0].debug
+
+
+@pytest.mark.skipif(
+    version.parse(scrapy.__version__) < version.parse("2.6"),
+    reason="Scrapy<2.6 uses optparse"
+)
+def test_argparse(command):
+    parser = ArgumentParser()
+    command.add_options(parser)
+    options = parser.parse_args(["--debug"])
+    assert options.debug