CVE-2021-41124 (Python, published 2021-10-05T21:15Z)
Description: value: Scrapy-splash is a library which provides Scrapy and JavaScript integration. In affected versions users who use [`HttpAuthMiddleware`](http://doc.scrapy.org/en/latest/topics/downloader-middleware.html#module-scrapy.downloadermiddlewares.httpauth) (i.e. the `http_user` and `http_pass` spider attributes) for Splash authentication will have any non-Splash request expose your credentials to the request target. This includes `robots.txt` requests sent by Scrapy when the `ROBOTSTXT_OBEY` setting is set to `True`. Upgrade to scrapy-splash 0.8.0 and use the new `SPLASH_USER` and `SPLASH_PASS` settings instead to set your Splash authentication credentials safely. If you cannot upgrade, set your Splash request credentials on a per-request basis, [using the `splash_headers` request parameter](https://github.com/scrapy-plugins/scrapy-splash/tree/0.8.x#http-basic-auth), instead of defining them globally using the [`HttpAuthMiddleware`](http://doc.scrapy.org/en/latest/topics/downloader-middleware.html#module-scrapy.downloadermiddlewares.httpauth). Alternatively, make sure all your requests go through Splash. That includes disabling the [robots.txt middleware](https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#topics-dlmw-robots).
Severity: MEDIUM (CVSS v3 base score: 7.5)
Weakness: CWE-200 \u2014 Exposure of Sensitive Information to an Unauthorized Actor
The product exposes sensitive information to an actor that is not explicitly authorized to have access to that information.
Repository: https://github.com/scrapy-plugins/scrapy-splash
Files changed:
CHANGES.rst
README.rst
example/scrashtest/settings.py
scrapy_splash/middleware.py
tests/conftest.py
tests/resources.py
tests/test_integration.py
tests/test_middleware.py
tests/utils.py
Commit message: Implement SPLASH_USER and SPLASH_PASS



Vulnerable code:
If you need HTTP Basic Authentication to access Splash, use
Scrapy's HttpAuthMiddleware_.
to ``splash_headers`` if HttpAuthMiddleware doesn't fit for some reason.
from scrapy.exceptions import NotConfigured
    def __init__(self, crawler, splash_base_url, slot_policy, log_400):
        splash_base_url = crawler.settings.get('SPLASH_URL',
                                               cls.default_splash_url)
        log_400 = crawler.settings.getbool('SPLASH_LOG_400', True)
        slot_policy = crawler.settings.get('SPLASH_SLOT_POLICY',
                                           cls.default_policy)
        return cls(crawler, splash_base_url, slot_policy, log_400)
            logger.warning(
                "SplashMiddleware; %(request)s will be handled without Splash",
            return request
        splash_options = request.meta['splash']
from scrapy.settings import Settings
def settings(request):
    return Settings(s)
from twisted.web.resource import Resource
from .utils import crawl_items, requires_splash, HtmlResource
  local wait = tonumber(splash.args.wait or 0.5)  
class HelloWorld(HtmlResource):
    html = """
    <html><body><script>document.write('hello world!');</script></body></html>
    """
    extra_headers = {'X-MyHeader': 'my value', 'Set-Cookie': 'sessionid=ABCD'}
class Http400Resource(HtmlResource):
    status_code = 400
    html = "Website returns HTTP 400 error"
class ManyCookies(Resource, object):
    class SetMyCookie(HtmlResource):
        html = "hello!"
        extra_headers = {'Set-Cookie': 'login=1'}
    def __init__(self):
        super(ManyCookies, self).__init__()
        self.putChild(b'', HelloWorld())
        self.putChild(b'login', self.SetMyCookie())
class ResponseSpider(scrapy.Spider):
    """ Make a request to URL, return Scrapy response """
    url = None
        yield SplashRequest(self.url)
    def parse(self, response):
        yield {'response': response}
    assert len(items) == 1
    resp = items[0]['response']
    assert len(items) == 1
    resp = items[0]['response']
        custom_settings = {'HTTPERROR_ALLOW_ALL': True}

    class GoodRequestSpider(ResponseSpider):
        custom_settings = {'HTTPERROR_ALLOW_ALL': True}

        def start_requests(self):
            yield SplashRequest(self.url, endpoint='execute',
                                args={'lua_source': DEFAULT_SCRIPT})


    resp = items[0]['response']
    items, url, crawler = yield crawl_items(GoodRequestSpider, Http400Resource,
    resp = items[0]['response']
def _get_mw():
    crawler = _get_crawler({})
    req2 = mw.process_request(req2, None)
    req = mw.process_request(req, None)
    req2 = mw.process_request(req, None)
    req = mw.process_request(req, None)
    req = mw.process_request(req, None)
    req = mw.process_request(req, None)
    req = mw.process_request(req, spider)
    req = mw.process_request(req, spider)
    req = scrapy.Request("http://example.com", meta = {'splash': {
    req1 = scrapy.Request("http://example.com", meta = {
    req2 = scrapy.Request("http://example.com", meta = {
from twisted.web.resource import Resource
from scrapy_splash.utils import to_bytes
from tests.mockserver import MockServer
class HtmlResource(Resource):
    isLeaf = True
    content_type = 'text/html'
    html = ''
    extra_headers = {}
    status_code = 200

    def render_GET(self, request):
        request.setHeader(b'content-type', to_bytes(self.content_type))
        for name, value in self.extra_headers.items():
            request.setHeader(to_bytes(name), to_bytes(value))
        request.setResponseCode(self.status_code)
        return to_bytes(self.html)


def crawl_items(spider_cls, resource_cls, settings, spider_kwargs=None):
        root_url = s.root_url
    result = crawler.spider.collected_items, s.root_url, crawler

Fixed code:
*   **Security bug fix:**

    If you use :ref:`HttpAuthMiddleware` (i.e. the ``http_user`` and
    ``http_pass`` spider attributes) for Splash authentication, any non-Splash
    request will expose your credentials to the request target. This includes
    ``robots.txt`` requests sent by Scrapy when the ``ROBOTSTXT_OBEY`` setting
    is set to ``True``.

    Use the new ``SPLASH_USER`` and ``SPLASH_PASS`` settings instead to set
    your Splash authentication credentials safely.

    .. _HttpAuthMiddleware: http://doc.scrapy.org/en/latest/topics/downloader-middleware.html#module-scrapy.downloadermiddlewares.httpauth

If you need to use HTTP Basic Authentication to access Splash, use the
``SPLASH_USER`` and ``SPLASH_PASS`` optional settings::

    SPLASH_USER = 'user'
    SPLASH_PASS = 'userpass'
to ``splash_headers`` if you want to change credentials per-request::

    import scrapy
    from w3lib.http import basic_auth_header

    class MySpider(scrapy.Spider):
        # ...
        def start_requests(self):
            auth = basic_auth_header('user', 'userpass')
            yield SplashRequest(url, self.parse,
                                splash_headers={'Authorization': auth})

**WARNING:** Don't use :ref:`HttpAuthMiddleware`
(i.e. ``http_user`` / ``http_pass`` spider attributes) for Splash
authentication: if you occasionally send a non-Splash request from your spider,
you may expose Splash credentials to a remote website, as HttpAuthMiddleware
sets credentials for all requests unconditionally.
ROBOTSTXT_OBEY = True
from w3lib.http import basic_auth_header
from scrapy.exceptions import NotConfigured, IgnoreRequest
from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware
    def __init__(self, crawler, splash_base_url, slot_policy, log_400, auth):
        self.auth = auth
        s = crawler.settings
        splash_base_url = s.get('SPLASH_URL', cls.default_splash_url)
        log_400 = s.getbool('SPLASH_LOG_400', True)
        slot_policy = s.get('SPLASH_SLOT_POLICY', cls.default_policy)
        splash_user = s.get('SPLASH_USER', '')
        splash_pass = s.get('SPLASH_PASS', '')
        auth = None
        if splash_user or splash_pass:
            auth = basic_auth_header(splash_user, splash_pass)
        return cls(crawler, splash_base_url, slot_policy, log_400, auth)
        if _http_auth_enabled(spider):
            replace_downloader_middleware(self.crawler, RobotsTxtMiddleware,
                                          SafeRobotsTxtMiddleware)
        splash_options = request.meta['splash']
            logger.error(
                "SplashMiddleware; %(request)s is dropped",
            self.crawler.stats.inc_value('splash/dropped/method/{}'.format(
                request.method))
            raise IgnoreRequest("SplashRequest doesn't support "
                                "HTTP {} method".format(request.method))
                # Headers set by HttpAuthMiddleware should be used for Splash,
                # not for the remote website (backwards compatibility).
                if _http_auth_enabled(spider):
                    headers.pop('Authorization', None)
        if self.auth is not None:
            headers['Authorization'] = self.auth
        new_request.meta['dont_obey_robotstxt'] = True


class SafeRobotsTxtMiddleware(RobotsTxtMiddleware):
    def process_request(self, request, spider):
        # disable robots.txt for Splash requests
        if _http_auth_enabled(spider) and 'splash' in request.meta:
            return
        return super(SafeRobotsTxtMiddleware, self).process_request(
            request, spider)


def _http_auth_enabled(spider):
    # FIXME: this function should always return False if HttpAuthMiddleware is
    # not in a middleware list.
    return getattr(spider, 'http_user', '') or getattr(spider, 'http_pass', '')


def replace_downloader_middleware(crawler, old_cls, new_cls):
    """ Replace downloader middleware with another one """
    try:
        new_mw = new_cls.from_crawler(crawler)
    except NotConfigured:
        return

    mw_manager = crawler.engine.downloader.middleware
    mw_manager.middlewares = tuple([
        mw if mw.__class__ is not old_cls else new_mw
        for mw in mw_manager.middlewares
    ])
    for method_name, callbacks in mw_manager.methods.items():
        for idx, meth in enumerate(callbacks):
            method_cls = meth.__self__.__class__
            if method_cls is old_cls:
                new_meth = getattr(new_mw, method_name)
                # logger.debug("{} is replaced with {}".format(meth, new_meth))
                callbacks[idx] = new_meth
from .mockserver import MockServer
from .resources import SplashProtected
def settings():
    return s
@pytest.fixture()
def settings_auth(settings):
    with MockServer(SplashProtected) as s:
        print("splash url:", s.root_url)
        settings['SPLASH_URL'] = s.root_url
        yield settings
# -*- coding: utf-8 -*-
import os
from six.moves.urllib.parse import urlparse

from twisted.web.resource import Resource
from zope.interface import implementer
from twisted.web import resource, guard, proxy
from twisted.cred.portal import IRealm, Portal
from twisted.cred.checkers import InMemoryUsernamePasswordDatabaseDontUse

from scrapy_splash.utils import to_bytes


class HtmlResource(Resource):
    isLeaf = True
    content_type = 'text/html'
    html = ''
    extra_headers = {}
    status_code = 200

    def render_GET(self, request):
        request.setHeader(b'content-type', to_bytes(self.content_type))
        for name, value in self.extra_headers.items():
            request.setHeader(to_bytes(name), to_bytes(value))
        request.setResponseCode(self.status_code)
        return to_bytes(self.html)


class HelloWorld(HtmlResource):
    html = """
    <html><body><script>document.write('hello world!');</script></body></html>
    """
    extra_headers = {'X-MyHeader': 'my value', 'Set-Cookie': 'sessionid=ABCD'}


class HelloWorldDisallowByRobots(HelloWorld):
    """ Disallow itself via robots.txt """
    isLeaf = False

    def getChild(self, name, request):
        if name == b"robots.txt":
            return self.RobotsTxt()
        return self

    class RobotsTxt(Resource):
        isLeaf = True
        def render_GET(self, request):
            return b'User-Agent: *\nDisallow: /\n'


class HelloWorldDisallowAuth(HelloWorldDisallowByRobots):
    """ Disallow itself via robots.txt if a request to robots.txt
    contains basic auth header. """
    class RobotsTxt(HelloWorldDisallowByRobots.RobotsTxt):
        def render_GET(self, request):
            if request.requestHeaders.hasHeader('Authorization'):
                return super(HelloWorldDisallowAuth.RobotsTxt, self).render_GET(request)
            request.setResponseCode(404)
            return b''


class Http400Resource(HtmlResource):
    status_code = 400
    html = "Website returns HTTP 400 error"


class ManyCookies(Resource, object):
    class SetMyCookie(HtmlResource):
        html = "hello!"
        extra_headers = {'Set-Cookie': 'login=1'}

    def __init__(self):
        super(ManyCookies, self).__init__()
        self.putChild(b'', HelloWorld())
        self.putChild(b'login', self.SetMyCookie())


def splash_proxy():
    splash_url = os.environ.get('SPLASH_URL')
    p = urlparse(splash_url)
    return lambda: proxy.ReverseProxyResource(p.hostname, int(p.port), b'')


def password_protected(resource_cls, username, password):
    # Sorry, but this is nuts. A zillion of classes, arbitrary
    # unicode / bytes requirements at random places. Is there a simpler
    # way to get HTTP Basic Auth working in Twisted?
    @implementer(IRealm)
    class SimpleRealm(object):
        def requestAvatar(self, avatarId, mind, *interfaces):
            if resource.IResource in interfaces:
                return resource.IResource, resource_cls(), lambda: None
            raise NotImplementedError()

    creds = {username: password}
    checkers = [InMemoryUsernamePasswordDatabaseDontUse(**creds)]
    return lambda: guard.HTTPAuthSessionWrapper(
        Portal(SimpleRealm(), checkers),
        [guard.BasicCredentialFactory(b'example.com')])


HelloWorldProtected = password_protected(HelloWorld, 'user', b'userpass')
HelloWorldProtected.__name__ = 'HelloWorldProtected'
HelloWorldProtected.__module__ = __name__

SplashProtected = password_protected(splash_proxy(), 'user', b'userpass')
SplashProtected.__name__ = 'SplashProtected'
SplashProtected.__module__ = __name__
import pytest
from pkg_resources import parse_version
from w3lib.http import basic_auth_header
from .utils import crawl_items, requires_splash
from .resources import (
    HelloWorld,
    Http400Resource,
    ManyCookies,
    HelloWorldProtected,
    HelloWorldDisallowByRobots,
    HelloWorldDisallowAuth,
)

  local wait = 0.01
  if splash.args.wait ~= nil then
    wait = splash.args.wait
  end
class ResponseSpider(scrapy.Spider):
    """ Make a request to URL, return Scrapy response """
    custom_settings = {
        'HTTPERROR_ALLOW_ALL': True,
        'ROBOTSTXT_OBEY': True,
    }
    url = None
    def start_requests(self):
        yield SplashRequest(self.url)
    def parse(self, response):
        yield {'response': response}
class LuaSpider(ResponseSpider):
    """ Make a request to URL using default Lua script """
    headers = None
    splash_headers = None
    def start_requests(self):
        yield SplashRequest(self.url,
                            endpoint='execute',
                            args={'lua_source': DEFAULT_SCRIPT},
                            headers=self.headers,
                            splash_headers=self.splash_headers)
class ScrapyAuthSpider(LuaSpider):
    """ Spider with incorrect (old, insecure) auth method """
    http_user = 'user'
    http_pass = 'userpass'
class NonSplashSpider(ResponseSpider):
    """ Spider which uses HTTP auth and doesn't use Splash """
    http_user = 'user'
    http_pass = 'userpass'
        yield scrapy.Request(self.url)

def assert_single_response(items):
    assert len(items) == 1
    return items[0]['response']
    resp = assert_single_response(items)
    resp = assert_single_response(items)
    resp = assert_single_response(items)
    items, url, crawler = yield crawl_items(LuaSpider, Http400Resource,
    resp = assert_single_response(items)


@requires_splash
@inlineCallbacks
def test_access_http_auth(settings):
    # website is protected
    items, url, crawler = yield crawl_items(LuaSpider, HelloWorldProtected,
                                            settings)
    response = assert_single_response(items)
    assert response.status == 401
    assert response.splash_response_status == 200

    # header can be used to access it
    AUTH_HEADERS = {'Authorization': basic_auth_header('user', 'userpass')}
    kwargs = {'headers': AUTH_HEADERS}
    items, url, crawler = yield crawl_items(LuaSpider, HelloWorldProtected,
                                            settings, kwargs)
    response = assert_single_response(items)
    assert 'hello' in response.body_as_unicode()
    assert response.status == 200
    assert response.splash_response_status == 200


@requires_splash
@inlineCallbacks
def test_protected_splash_no_auth(settings_auth):
    items, url, crawler = yield crawl_items(LuaSpider, HelloWorld,
                                            settings_auth)
    response = assert_single_response(items)
    assert 'Unauthorized' in response.body_as_unicode()
    assert 'hello' not in response.body_as_unicode()
    assert response.status == 401
    assert response.splash_response_status == 401


@requires_splash
@inlineCallbacks
def test_protected_splash_manual_headers_auth(settings_auth):
    AUTH_HEADERS = {'Authorization': basic_auth_header('user', 'userpass')}
    kwargs = {'splash_headers': AUTH_HEADERS}

    # auth via splash_headers should work
    items, url, crawler = yield crawl_items(LuaSpider, HelloWorld,
                                            settings_auth, kwargs)
    response = assert_single_response(items)
    assert 'hello' in response.body_as_unicode()
    assert response.status == 200
    assert response.splash_response_status == 200

    # but only for Splash, not for a remote website
    items, url, crawler = yield crawl_items(LuaSpider, HelloWorldProtected,
                                            settings_auth, kwargs)
    response = assert_single_response(items)
    assert 'hello' not in response.body_as_unicode()
    assert response.status == 401
    assert response.splash_response_status == 200


@requires_splash
@inlineCallbacks
def test_protected_splash_settings_auth(settings_auth):
    settings_auth['SPLASH_USER'] = 'user'
    settings_auth['SPLASH_PASS'] = 'userpass'

    # settings works
    items, url, crawler = yield crawl_items(LuaSpider, HelloWorld,
                                            settings_auth)
    response = assert_single_response(items)
    assert 'Unauthorized' not in response.body_as_unicode()
    assert 'hello' in response.body_as_unicode()
    assert response.status == 200
    assert response.splash_response_status == 200

    # they can be overridden via splash_headers
    bad_auth = {'splash_headers': {'Authorization': 'foo'}}
    items, url, crawler = yield crawl_items(LuaSpider, HelloWorld,
                                            settings_auth, bad_auth)
    response = assert_single_response(items)
    assert response.status == 401
    assert response.splash_response_status == 401

    # auth error on remote website
    items, url, crawler = yield crawl_items(LuaSpider, HelloWorldProtected,
                                            settings_auth)
    response = assert_single_response(items)
    assert response.status == 401
    assert response.splash_response_status == 200

    # auth both for Splash and for the remote website
    REMOTE_AUTH = {'Authorization': basic_auth_header('user', 'userpass')}
    remote_auth_kwargs = {'headers': REMOTE_AUTH}
    items, url, crawler = yield crawl_items(LuaSpider, HelloWorldProtected,
                                            settings_auth, remote_auth_kwargs)
    response = assert_single_response(items)
    assert response.status == 200
    assert response.splash_response_status == 200
    assert 'hello' in response.body_as_unicode()

    # enable remote auth, but not splash auth - request should fail
    del settings_auth['SPLASH_USER']
    del settings_auth['SPLASH_PASS']
    items, url, crawler = yield crawl_items(LuaSpider,
                                            HelloWorldProtected,
                                            settings_auth, remote_auth_kwargs)
    response = assert_single_response(items)
    assert response.status == 401
    assert response.splash_response_status == 401


@requires_splash
@inlineCallbacks
def test_protected_splash_httpauth_middleware(settings_auth):
    # httpauth middleware should enable auth for Splash, for backwards
    # compatibility reasons
    items, url, crawler = yield crawl_items(ScrapyAuthSpider, HelloWorld,
                                            settings_auth)
    response = assert_single_response(items)
    assert 'Unauthorized' not in response.body_as_unicode()
    assert 'hello' in response.body_as_unicode()
    assert response.status == 200
    assert response.splash_response_status == 200

    # but not for a remote website
    items, url, crawler = yield crawl_items(ScrapyAuthSpider,
                                            HelloWorldProtected,
                                            settings_auth)
    response = assert_single_response(items)
    assert 'hello' not in response.body_as_unicode()
    assert response.status == 401
    assert response.splash_response_status == 200

    # headers shouldn't be sent to robots.txt file
    items, url, crawler = yield crawl_items(ScrapyAuthSpider,
                                            HelloWorldDisallowAuth,
                                            settings_auth)
    response = assert_single_response(items)
    assert 'hello' in response.body_as_unicode()
    assert response.status == 200
    assert response.splash_response_status == 200

    # httpauth shouldn't be disabled for non-Splash requests
    items, url, crawler = yield crawl_items(NonSplashSpider,
                                            HelloWorldProtected,
                                            settings_auth)
    response = assert_single_response(items)
    assert 'hello' in response.body_as_unicode()
    assert response.status == 200
    assert not hasattr(response, 'splash_response_status')


@pytest.mark.xfail(
    parse_version(scrapy.__version__) < parse_version("1.1"),
    reason="https://github.com/scrapy/scrapy/issues/1471",
    strict=True,
    run=True,
)
@requires_splash
@inlineCallbacks
def test_robotstxt_can_work(settings_auth):

    def assert_robots_disabled(items):
        response = assert_single_response(items)
        assert response.status == response.splash_response_status == 200
        assert b'hello' in response.body

    def assert_robots_enabled(items, crawler):
        assert len(items) == 0
        assert crawler.stats.get_value('downloader/exception_type_count/scrapy.exceptions.IgnoreRequest') == 1

    def _crawl_items(spider, resource):
        return crawl_items(
            spider,
            resource,
            settings_auth,
            url_path='/',  # https://github.com/scrapy/protego/issues/17
        )

    # when old auth method is used, robots.txt should be disabled
    items, url, crawler = yield _crawl_items(ScrapyAuthSpider,
                                             HelloWorldDisallowByRobots)
    assert_robots_disabled(items)

    # but robots.txt should still work for non-Splash requests
    items, url, crawler = yield _crawl_items(NonSplashSpider,
                                             HelloWorldDisallowByRobots)
    assert_robots_enabled(items, crawler)

    # robots.txt should work when a proper auth method is used
    settings_auth['SPLASH_USER'] = 'user'
    settings_auth['SPLASH_PASS'] = 'userpass'
    items, url, crawler = yield _crawl_items(LuaSpider,
                                             HelloWorldDisallowByRobots)
    assert_robots_enabled(items, crawler)

    # disable robotstxt middleware - robots middleware shouldn't work
    class DontObeyRobotsSpider(LuaSpider):
        custom_settings = {
            'HTTPERROR_ALLOW_ALL': True,
            'ROBOTSTXT_OBEY': False,
        }
    items, url, crawler = yield _crawl_items(DontObeyRobotsSpider,
                                             HelloWorldDisallowByRobots)
    assert_robots_disabled(items)

    # disable robotstxt middleware via request meta
    class MetaDontObeyRobotsSpider(ResponseSpider):
        def start_requests(self):
            yield SplashRequest(self.url,
                                endpoint='execute',
                                meta={'dont_obey_robotstxt': True},
                                args={'lua_source': DEFAULT_SCRIPT})

    items, url, crawler = yield _crawl_items(MetaDontObeyRobotsSpider,
                                             HelloWorldDisallowByRobots)
    assert_robots_disabled(items)
def _get_mw(settings_dict=None):
    crawler = _get_crawler(settings_dict or {})

    req2 = mw.process_request(req2, None) or req2

    assert req2.meta['ajax_crawlable'] is True
    req = mw.process_request(req, None) or req
    req2 = mw.process_request(req, None) or req
    req = mw.process_request(req, None) or req
    req = mw.process_request(req, None) or req
    req = mw.process_request(req, None) or req
    req = mw.process_request(req, spider) or req
    req = mw.process_request(req, spider) or req
    req = mw.process_request(req, None) or req
    req = mw.process_request(req, None) or req
    req = mw.process_request(req, None) or req
    req = scrapy.Request("http://example.com", meta={'splash': {
    req1 = scrapy.Request("http://example.com", meta={
    req2 = scrapy.Request("http://example.com", meta={


def test_auth():
    def assert_auth_header(user, pwd, header):
        mw = _get_mw({'SPLASH_USER': user, 'SPLASH_PASS': pwd})
        req = mw.process_request(SplashRequest("http://example.com"), None)
        assert 'Authorization' in req.headers
        assert req.headers['Authorization'] == header

    def assert_no_auth_header(user, pwd):
        if user is not None or pwd is not None:
            mw = _get_mw({'SPLASH_USER': user, 'SPLASH_PASS': pwd})
        else:
            mw = _get_mw()
        req = mw.process_request(SplashRequest("http://example.com"), None)
        assert 'Authorization' not in req.headers

    assert_auth_header('root', '', b'Basic cm9vdDo=')
    assert_auth_header('root', 'pwd', b'Basic cm9vdDpwd2Q=')
    assert_auth_header('', 'pwd', b'Basic OnB3ZA==')

    assert_no_auth_header('', '')
    assert_no_auth_header(None, None)
from .mockserver import MockServer
def crawl_items(
    spider_cls,
    resource_cls,
    settings,
    spider_kwargs=None,
    url_path="",
):
        print("mock server", s.root_url)
        root_url = s.root_url + url_path
    items = getattr(crawler.spider, 'collected_items', [])
    result = items, root_url, crawler