Repository: jina-ai/jina
File: tests/integration/monitoring/test_monitoring.py (Python)

Code before:
        p.start()
        p.join()
        time.sleep(2)
        _assert_pending_value('0.0', runtime_name, port_monitoring)
    except Exception:
        assert False
    finally:  # clean up runtimes
        gateway_process.terminate()
        gateway_process.join()


def test_failed_successful_request_count(port_generator, failing_executor):
    port0 = port_generator()
    port1 = port_generator()

    with Flow(monitoring=True, port_monitoring=port0).add(
        uses=failing_executor, port_monitoring=port1
    ) as f:
        resp = req.get(f'http://localhost:{port1}/')
        assert resp.status_code == 200

        f.post(
            '/',
            inputs=DocumentArray.empty(size=10),
            request_size=1,
            continue_on_error=True,
        )  # send 10 requests, 5 should fail and 5 should succeed

        resp = req.get(f'http://localhost:{port1}/')

        assert (
            f'jina_number_of_successful_requests{{runtime_name="executor0/rep-0"}} 5.0'
            in str(resp.content)
        )

        assert (
            f'jina_number_of_failed_requests{{runtime_name="executor0/rep-0"}} 5.0'
            in str(resp.content)
        )

        resp = req.get(f'http://localhost:{port0}/')

        assert (
            f'jina_number_of_successful_requests{{runtime_name="gateway/rep-0/GRPCGatewayRuntime"}} 5.0'
            in str(resp.content)
        )

        assert (
            f'jina_number_of_failed_requests{{runtime_name="gateway/rep-0/GRPCGatewayRuntime"}} 5.0'
            in str(resp.content)
        )

Review comment (bug):
This tests for Executors that fail, but that is only one failure mode. For example, the network connection itself could fail, which produces a different (gRPC level) error. I believe that your implementation will also handle that, but I am not 100% sure. In either case, a test where the network itself fails that would be nice. You can look at [this](https://github.com/jina-ai/jina/blob/master/tests/integration/runtimes/test_network_failures.py) file for inspiration.

Another failure case are timeouts between gateway and Executor, are those also handled?

Code after:
    gateway_process = _create_gateway(
        gateway_port, port_monitoring, graph_description, pod_addresses, protocol
    )

    time.sleep(1.0)

    AsyncNewLoopRuntime.wait_for_ready_or_shutdown(
        timeout=5.0,
        ctrl_address=f'0.0.0.0:{gateway_port}',
        ready_or_shutdown_event=multiprocessing.Event(),
    )

    try:
        p = multiprocessing.Process(target=_send_request, args=(gateway_port, protocol))
        p.start()
        p.join()
        time.sleep(2)
        _assert_pending_value('0.0', runtime_name, port_monitoring)
    except Exception:
        assert False
    finally:  # clean up runtimes
        gateway_process.terminate()
        gateway_process.join()


def test_failed_successful_request_count(port_generator, failing_executor):
    port0 = port_generator()
    port1 = port_generator()

    with Flow(monitoring=True, port_monitoring=port0).add(
        uses=failing_executor, port_monitoring=port1
    ) as f:
        resp = req.get(f'http://localhost:{port1}/')
        assert resp.status_code == 200

        f.post(
            '/fail',
            inputs=DocumentArray.empty(size=10),
            request_size=1,
            continue_on_error=True,
        )  # send 10 requests, 5 should fail and 5 should succeed

        resp = req.get(f'http://localhost:{port1}/')

        assert (
            f'jina_successful_requests_total{{runtime_name="executor0/rep-0"}} 5.0'
            in str(resp.content)
        )

        assert (
            f'jina_failed_requests_total{{runtime_name="executor0/rep-0"}} 5.0'