Repository: scrapinghub/spidermon (language: python)

Issue:
Built-in monitor to compare item count with previous jobs
This monitor is more useful when we are running the spider in [Scrapy Cloud](https://www.zyte.com/scrapy-cloud/). The objective is to compare the number of extracted items in current job with the number of items extracted in previous jobs.

- Get only `finished` jobs.
- Limit the number of jobs to be considered (configurable)
- if possible add more ways to decide which jobs to consider, not only the finished ones. This is useful when we have a `finished` job, but we want to exclude that execution for some reason (it failed y other reason for example)
- Configurable percentage to allow variations ("require that this job has at least 90% of items from the average 10 last jobs)
Add `JobsComparisonMonitor`
closes #301

It adds the `JobComparisonMonitor`, for checking for drops in scraped item count compared to previous jobs, when running jobs in [Scrapy Cloud](https://www.zyte.com/scrapy-cloud).
- allow filtering by jobs states (eg.: `finished`, `running`, `closespider_timeout`, etc)
- allow filtering by arbitrary tags 
- also updates the module that holds a singleton to the Scrapy Cloud API client.

----
TODO:
- [x] use scrapinghubclient instead
- [x] inherit from `BaseStatMonitor`
- [x] filter by tags
Add JobsComparisonMonitor
Closes #301.
Continuing #323 (resolves #371).

- Allow filtering by jobs states (eg.: finished, running, closespider_timeout, etc);
- Allow filtering by arbitrary tags.

NOTE: Added support to allow for more than 1000 jobs to be queried.


Discussion / hints:
Hi guys, please consider job comparison based on 
1. tags. It can be one or several tags
2. arguments. It can be all arguments or several arguments (if it's possible)




Fix patch:
diff --git a/docs/source/monitors.rst b/docs/source/monitors.rst
index 08b02087..55931b6f 100644
--- a/docs/source/monitors.rst
+++ b/docs/source/monitors.rst
@@ -151,10 +151,10 @@ The Basic Monitors
 Spidermon has some batteries included :)
 
 .. automodule:: spidermon.contrib.scrapy.monitors
-    :members: ItemCountMonitor, ErrorCountMonitor, WarningCountMonitor, FinishReasonMonitor,
+    :members: ItemCountMonitor, ErrorCountMonitor, WarningCountMonitor, CriticalCountMonitor, FinishReasonMonitor,
               UnwantedHTTPCodesMonitor, ItemValidationMonitor, FieldCoverageMonitor,
               RetryCountMonitor, DownloaderExceptionMonitor, SuccessfulRequestsMonitor,
-              TotalRequestsMonitor, PeriodicExecutionTimeMonitor
+              TotalRequestsMonitor, PeriodicExecutionTimeMonitor, ZyteJobsComparisonMonitor
 
 Is there a **Basic Scrapy Suite** ready to use?
 ------------------------------------------------
diff --git a/spidermon/contrib/scrapy/monitors.py b/spidermon/contrib/scrapy/monitors.py
index 2cb8a81c..5c4f56b4 100644
--- a/spidermon/contrib/scrapy/monitors.py
+++ b/spidermon/contrib/scrapy/monitors.py
@@ -1,7 +1,11 @@
 import datetime
+import json
+import math
+import os
 
 from spidermon import Monitor, MonitorSuite, monitors
 from spidermon.exceptions import NotConfigured
+from spidermon.utils import zyte
 from spidermon.utils.settings import getdictorlist
 
 from ..monitors.mixins.spider import SpiderMonitorMixin, StatsMonitorMixin
@@ -13,6 +17,10 @@
 SPIDERMON_MAX_RETRIES = "SPIDERMON_MAX_RETRIES"
 SPIDERMON_MIN_SUCCESSFUL_REQUESTS = "SPIDERMON_MIN_SUCCESSFUL_REQUESTS"
 SPIDERMON_MAX_REQUESTS_ALLOWED = "SPIDERMON_MAX_REQUESTS_ALLOWED"
+SPIDERMON_JOBS_COMPARISON = "SPIDERMON_JOBS_COMPARISON"
+SPIDERMON_JOBS_COMPARISON_STATES = "SPIDERMON_JOBS_COMPARISON_STATES"
+SPIDERMON_JOBS_COMPARISON_TAGS = "SPIDERMON_JOBS_COMPARISON_TAGS"
+SPIDERMON_JOBS_COMPARISON_THRESHOLD = "SPIDERMON_JOBS_COMPARISON_THRESHOLD"
 
 
 class BaseScrapyMonitor(Monitor, SpiderMonitorMixin):
@@ -530,6 +538,113 @@ def test_execution_time(self):
         self.assertLess(duration.total_seconds(), max_execution_time, msg=msg)
 
 
+@monitors.name("Jobs Comparison Monitor")
+class ZyteJobsComparisonMonitor(BaseStatMonitor):
+    """
+    .. note::
+       This monitor is useful when running jobs in
+       `Zyte's Scrapy Cloud <https://www.zyte.com/scrapy-cloud/>`_.
+
+    Check for a drop in scraped item count compared to previous jobs.
+
+    You need to set the number of previous jobs to compare, using ``SPIDERMON_JOBS_COMPARISON``.
+    The default is ``0`` which disables the monitor. We use the average of the scraped items count.
+
+    You can configure which percentage of the previous item count is the minimum acceptable, by
+    using the setting ``SPIDERMON_JOBS_COMPARISON_THRESHOLD``. We expect a float number between
+    ``0.0`` (not inclusive) and with no upper limit (meaning we can check if itemcount is increasing
+    at a certain rate). If not set, a NotConfigured error will be raised.
+
+    You can filter which jobs to compare based on their states using the
+    ``SPIDERMON_JOBS_COMPARISON_STATES`` setting. The default value is ``("finished",)``.
+
+    You can also filter which jobs to compare based on their tags using the
+    ``SPIDERMON_JOBS_COMPARISON_TAGS`` setting. Among the defined tags we consider only those
+    that are also present in the current job.
+    """
+
+    stat_name = "item_scraped_count"
+    assert_type = ">="
+
+    def run(self, result):
+
+        if (
+            SPIDERMON_JOBS_COMPARISON not in self.crawler.settings.attributes
+            or self.crawler.settings.getint(SPIDERMON_JOBS_COMPARISON) <= 0
+        ):
+            raise NotConfigured(
+                f"Configure SPIDERMON_JOBS_COMPARISON to your project "
+                f"settings to use {self.monitor_name}."
+            )
+
+        if (
+            SPIDERMON_JOBS_COMPARISON_THRESHOLD not in self.crawler.settings.attributes
+            or self.crawler.settings.getfloat(SPIDERMON_JOBS_COMPARISON_THRESHOLD) <= 0
+        ):
+            raise NotConfigured(
+                f"Configure SPIDERMON_JOBS_COMPARISON_THRESHOLD to your project "
+                f"settings to use {self.monitor_name}."
+            )
+
+        return super().run(result)
+
+    def _get_jobs(self, states, number_of_jobs):
+
+        tags = self._get_tags_to_filter()
+
+        jobs = []
+        start = 0
+        _jobs = zyte.client.spider.jobs.list(
+            start=start,
+            state=states,
+            count=number_of_jobs,
+            filters=dict(has_tag=tags) if tags else None,
+        )
+        while _jobs:
+            jobs.extend(_jobs)
+            start += 1000
+            _jobs = zyte.client.spider.jobs.list(
+                start=start,
+                state=states,
+                count=number_of_jobs,
+                filters=dict(has_tag=tags) if tags else None,
+            )
+        return jobs
+
+    def _get_tags_to_filter(self):
+        """
+        Return the intersect of the desired tags to filter and
+        the ones from the current job.
+        """
+        desired_tags = self.crawler.settings.getlist(SPIDERMON_JOBS_COMPARISON_TAGS)
+        if not desired_tags:
+            return {}
+
+        current_tags = json.loads(os.environ.get("SHUB_JOB_DATA", "{}")).get("tags")
+        if not current_tags:
+            return {}
+
+        tags_to_filter = set(desired_tags) & set(current_tags)
+        return sorted(tags_to_filter)
+
+    def get_threshold(self):
+
+        number_of_jobs = self.crawler.settings.getint(SPIDERMON_JOBS_COMPARISON)
+
+        threshold = self.crawler.settings.getfloat(SPIDERMON_JOBS_COMPARISON_THRESHOLD)
+
+        states = self.crawler.settings.getlist(
+            SPIDERMON_JOBS_COMPARISON_STATES, ("finished",)
+        )
+
+        jobs = self._get_jobs(states, number_of_jobs)
+
+        previous_count = sum(job.get("items", 0) for job in jobs) / len(jobs)
+
+        expected_item_extracted = math.ceil(previous_count * threshold)
+        return expected_item_extracted
+
+
 class SpiderCloseMonitorSuite(MonitorSuite):
     """This Monitor Suite implements the following monitors:
 


Test patch:
diff --git a/tests/contrib/scrapy/monitors/test_jobs_comparison_monitor.py b/tests/contrib/scrapy/monitors/test_jobs_comparison_monitor.py
new file mode 100644
index 00000000..978b4f43
--- /dev/null
+++ b/tests/contrib/scrapy/monitors/test_jobs_comparison_monitor.py
@@ -0,0 +1,153 @@
+from unittest.mock import Mock, call
+
+import pytest
+from spidermon import MonitorSuite
+from spidermon.contrib.scrapy import monitors
+from spidermon.contrib.scrapy.monitors import (
+    SPIDERMON_JOBS_COMPARISON,
+    SPIDERMON_JOBS_COMPARISON_STATES,
+    SPIDERMON_JOBS_COMPARISON_TAGS,
+    SPIDERMON_JOBS_COMPARISON_THRESHOLD,
+    ZyteJobsComparisonMonitor,
+)
+from spidermon.exceptions import NotConfigured
+
+
+@pytest.fixture
+def mock_jobs(previous_counts):
+    return Mock(return_value=[dict(items=c) for c in previous_counts])
+
+
+@pytest.fixture
+def mock_suite(mock_jobs, monkeypatch):
+    monkeypatch.setattr(ZyteJobsComparisonMonitor, "_get_jobs", mock_jobs)
+    return MonitorSuite(monitors=[ZyteJobsComparisonMonitor])
+
+
+@pytest.fixture
+def mock_suite_and_zyte_client(
+    monkeypatch,
+    number_of_jobs,
+):
+    def get_paginated_jobs(**kwargs):
+        start = kwargs["start"]
+        if start < number_of_jobs:
+            return [
+                Mock() for _ in range(start, max(number_of_jobs - 1000, number_of_jobs))
+            ]
+        return []
+
+    monkeypatch.setenv("SHUB_JOB_DATA", '{"tags":["tag1","tag2","tag3"]}')
+    monkeypatch.setattr(monitors, "zyte", Mock())
+    monitors.zyte.client.spider.jobs.list.side_effect = get_paginated_jobs
+
+    return MonitorSuite(monitors=[monitors.ZyteJobsComparisonMonitor]), monitors.zyte
+
+
+@pytest.mark.parametrize(
+    [
+        "number_of_jobs",
+        "threshold",
+        "item_count",
+        "previous_counts",
+        "expected_to_be_enabled",
+    ],
+    [
+        (0, 0, 1, [1], False),
+        (0, 0.5, 1, [1], False),
+        (1, 0, 1, [1], False),
+        (-1, 0.5, 1, [1], False),
+        (1, -0.5, 1, [1], False),
+        (-1, -0.5, 1, [1], False),
+        (1, 0.5, 1, [1], True),
+        (5, 0.5, 1, [1], True),
+        (5, 1.1, 1, [1], True),
+    ],
+)
+def test_jobs_comparison_monitor_is_enabled(
+    make_data, mock_suite, item_count, number_of_jobs, expected_to_be_enabled, threshold
+):
+    data = make_data(
+        {
+            SPIDERMON_JOBS_COMPARISON: number_of_jobs,
+            SPIDERMON_JOBS_COMPARISON_THRESHOLD: threshold,
+        }
+    )
+    data["stats"]["item_scraped_count"] = item_count
+    runner = data.pop("runner")
+
+    if expected_to_be_enabled:
+        try:
+            runner.run(mock_suite, **data)
+        except NotConfigured:
+            pytest.fail("RAISED NotConfigured exception")
+    else:
+        with pytest.raises(NotConfigured):
+            runner.run(mock_suite, **data)
+
+
+@pytest.mark.parametrize(
+    ["item_count", "previous_counts", "threshold", "should_raise"],
+    [
+        (90, [100], 0.9, False),
+        (80, [100, 101, 99], 0.8, False),
+        (90, [100, 100], 0.9, False),
+        (111, [100, 100], 1.1, False),
+        (100, [100, 100], 1.1, True),
+        (90, [100, 101], 0.9, True),
+        (80, [100, 101, 99, 102], 0.8, True),
+    ],
+)
+def test_jobs_comparison_monitor_threshold(
+    make_data, mock_suite, item_count, threshold, should_raise
+):
+    data = make_data(
+        {SPIDERMON_JOBS_COMPARISON: 1, SPIDERMON_JOBS_COMPARISON_THRESHOLD: threshold}
+    )
+    data["stats"]["item_scraped_count"] = item_count
+    runner = data.pop("runner")
+    runner.run(mock_suite, **data)
+
+    assert should_raise == bool(runner.result.monitor_results[0].error)
+
+
+@pytest.mark.parametrize(
+    ["states", "number_of_jobs", "tags", "threshold"],
+    [
+        (("finished",), 5, ("tag1", "tag2"), 0.5),
+        (("foo", "bar"), 10, ("tag3",), 0.5),
+        (("foo", "bar"), 1001, ("tag3",), 0.5),
+        (("foo", "bar"), 2000, ("tag3",), 0.5),
+    ],
+)
+def test_arguments_passed_to_zyte_client(
+    make_data,
+    mock_suite_and_zyte_client,
+    states,
+    number_of_jobs,
+    tags,
+    threshold,
+):
+    data = make_data(
+        {
+            SPIDERMON_JOBS_COMPARISON: number_of_jobs,
+            SPIDERMON_JOBS_COMPARISON_STATES: states,
+            SPIDERMON_JOBS_COMPARISON_TAGS: tags,
+            SPIDERMON_JOBS_COMPARISON_THRESHOLD: threshold,
+        }
+    )
+    suite, zyte_module = mock_suite_and_zyte_client
+    runner = data.pop("runner")
+    runner.run(suite, **data)
+
+    calls = [
+        call.zyte_module.client.spider.jobs.list(
+            start=n,
+            state=list(states),
+            count=number_of_jobs,
+            filters={"has_tag": list(tags)},
+        )
+        for n in range(0, number_of_jobs + 1000, 1000)
+    ]
+
+    zyte_module.client.spider.jobs.list.assert_has_calls(calls)