Repository: PrefectHQ/prefect

Issue:
Long log messages failed to be send
### First check

- [X] I added a descriptive title to this issue.
- [X] I used the GitHub search to find a similar issue and didn't find it.
- [X] I searched the Prefect documentation for this issue.
- [X] I checked that this issue is related to Prefect and not one of its dependencies.

### Bug summary

A task failed due to an error in a SQL statement. The SQL statement was very long (6mb) and did not appear in the prefect UI but if this limit is known, you can maybe truncate the log statement? The log messages directly before the SQL error were sent without any problems and appear in the prefect UI. But in the UI it seems that there were a few log messages and then the task state is set to "failed" without any clue why (I had to look in the docker logs on the server).

### Reproduction

```python3
/
```


### Error

Sensitive information was excluded with a `****`

```bash
10:07:52.183 | WARNING | Flow run 'sweet-capybara' - ****
10:07:54.138 | ERROR   | Flow run 'sweet-capybara' - Encountered exception during execution:
Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1900, in _execute_context
    self.dialect.do_execute(
  File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 736, in do_execute
    cursor.execute(statement, parameters)
psycopg2.errors.NotNullViolation: ****
DETAIL:  Failing row contains ****


The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/prefect/engine.py", line 643, in orchestrate_flow_run
    result = await run_sync(flow_call)
  File "/usr/local/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 156, in run_sync_in_interruptible_worker_thread
    tg.start_soon(
  File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 662, in __aexit__
    raise exceptions[0]
  File "/usr/local/lib/python3.9/site-packages/anyio/to_thread.py", line 31, in run_sync
    return await get_asynclib().run_sync_in_worker_thread(
  File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 937, in run_sync_in_worker_thread
    return await future
  File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 867, in run
    result = context.run(func, *args)
  File "/usr/local/lib/python3.9/site-packages/prefect/utilities/asyncutils.py", line 135, in capture_worker_thread_and_result
    result = __fn(*args, **kwargs)
  File "tasks/****", line 607, in run_prefect_flow
    run_flow(max_num=max_num, download=download)
  File "tasks/****", line 580, in run_flow
    import_endpoint(engine, model, download, max_num)
  File "tasks/****", line 183, in import_endpoint
    count = import_dump(engine, json_path, model, max_num)
  File "tasks/****", line 132, in import_dump
    pd.DataFrame(batch).to_sql(
  File "/usr/local/lib/python3.9/site-packages/pandas/core/generic.py", line 2987, in to_sql
    return sql.to_sql(
  File "/usr/local/lib/python3.9/site-packages/pandas/io/sql.py", line 695, in to_sql
    return pandas_sql.to_sql(
  File "/usr/local/lib/python3.9/site-packages/pandas/io/sql.py", line 1738, in to_sql
    total_inserted = sql_engine.insert_records(
  File "/usr/local/lib/python3.9/site-packages/pandas/io/sql.py", line 1335, in insert_records
    raise err
  File "/usr/local/lib/python3.9/site-packages/pandas/io/sql.py", line 1325, in insert_records
    return table.insert(chunksize=chunksize, method=method)
  File "/usr/local/lib/python3.9/site-packages/pandas/io/sql.py", line 946, in insert
    num_inserted = exec_insert(conn, keys, chunk_iter)
  File "/usr/local/lib/python3.9/site-packages/pandas/io/sql.py", line 869, in _execute_insert_multi
    result = conn.execute(stmt)
  File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1380, in execute
    return meth(self, multiparams, params, _EMPTY_EXECUTION_OPTS)
  File "/usr/local/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 334, in _execute_on_connection
    return connection._execute_clauseelement(
  File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1572, in _execute_clauseelement
    ret = self._execute_context(
  File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1943, in _execute_context
    self._handle_dbapi_exception(
  File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 2124, in _handle_dbapi_exception
    util.raise_(
  File "/usr/local/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 211, in raise_
    raise exception
  File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1900, in _execute_context
    self.dialect.do_execute(
  File "/usr/local/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 736, in do_execute
    cursor.execute(statement, parameters)
sqlalchemy.exc.IntegrityError: (psycopg2.errors.NotNullViolation) ****

[Very long SQL statement]

10:11:58.635 | ERROR   | Flow run 'sweet-capybara' - Crash detected! Request to https://api.prefect.cloud/api/accounts/****/workspaces/****/flow_runs/****/set_state failed: Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/anyio/streams/tls.py", line 130, in _call_sslobject_method
    result = func(*args)
  File "/usr/local/lib/python3.9/ssl.py", line 889, in read
    v = self._sslobj.read(len)
ssl.SSLWantReadError: The operation did not complete (read) (_ssl.c:2633)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/httpcore/backends/asyncio.py", line 34, in read
    return await self._stream.receive(max_bytes=max_bytes)
  File "/usr/local/lib/python3.9/site-packages/anyio/streams/tls.py", line 195, in receive
    data = await self._call_sslobject_method(self._ssl_object.read, max_bytes)
  File "/usr/local/lib/python3.9/site-packages/anyio/streams/tls.py", line 137, in _call_sslobject_method
    data = await self.transport_stream.receive()
  File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 1265, in receive
    await self._protocol.read_event.wait()
  File "/usr/local/lib/python3.9/asyncio/locks.py", line 226, in wait
    await fut
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/httpcore/_exceptions.py", line 10, in map_exceptions
    yield
  File "/usr/local/lib/python3.9/site-packages/httpcore/backends/asyncio.py", line 36, in read
    return b""
  File "/usr/local/lib/python3.9/site-packages/anyio/_core/_tasks.py", line 118, in __exit__
    raise TimeoutError
TimeoutError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 60, in map_httpcore_exceptions
    yield
  File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
    resp = await self._pool.handle_async_request(req)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection_pool.py", line 253, in handle_async_request
    raise exc
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection_pool.py", line 237, in handle_async_request
    response = await connection.handle_async_request(request)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection.py", line 90, in handle_async_request
    return await self._connection.handle_async_request(request)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 144, in handle_async_request
    raise exc
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 108, in handle_async_request
    await self._send_request_body(request=request, stream_id=stream_id)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 214, in _send_request_body
    max_flow = await self._wait_for_outgoing_flow(request, stream_id)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 386, in _wait_for_outgoing_flow
    await self._receive_events(request)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 281, in _receive_events
    events = await self._read_incoming_data(request)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 341, in _read_incoming_data
    raise exc
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 327, in _read_incoming_data
    data = await self._network_stream.read(self.READ_NUM_BYTES, timeout)
  File "/usr/local/lib/python3.9/site-packages/httpcore/backends/asyncio.py", line 36, in read
    return b""
  File "/usr/local/lib/python3.9/contextlib.py", line 137, in __exit__
    self.gen.throw(typ, value, traceback)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_exceptions.py", line 14, in map_exceptions
    raise to_exc(exc)
httpcore.ReadTimeout

The above exception was the direct cause of the following exception:

httpx.ReadTimeout
.
10:11:58.917 | ERROR   | prefect.engine - Engine execution of flow run '430b52b9-ad51-495c-8436-27ffd4126689' exited with unexpected exception
Traceback (most recent call last):
  File "/app/tasks/run.py", line 127, in <module>
    enter_flow_run_engine_from_subprocess(the_flow_run_id)
  File "/app/tasks/run.py", line 106, in enter_flow_run_engine_from_subprocess
    return anyio.run(retrieve_flow_then_begin_flow_run, flow_run_id)
  File "/usr/local/lib/python3.9/site-packages/anyio/_core/_eventloop.py", line 70, in run
    return asynclib.run(func, *args, **backend_options)
  File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 292, in run
    return native_run(wrapper(), debug=debug)
  File "/usr/local/lib/python3.9/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/usr/local/lib/python3.9/asyncio/base_events.py", line 647, in run_until_complete
    return future.result()
  File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 287, in wrapper
    return await func(*args)
  File "/usr/local/lib/python3.9/site-packages/prefect/client/utilities.py", line 47, in with_injected_client
    return await fn(*args, **kwargs)
  File "/app/tasks/run.py", line 92, in retrieve_flow_then_begin_flow_run
    return await begin_flow_run(
  File "/usr/local/lib/python3.9/site-packages/prefect/engine.py", line 379, in begin_flow_run
    terminal_or_paused_state = await orchestrate_flow_run(
  File "/usr/local/lib/python3.9/site-packages/prefect/engine.py", line 703, in orchestrate_flow_run
    state = await propose_state(
  File "/usr/local/lib/python3.9/site-packages/prefect/engine.py", line 1806, in propose_state
    response = await client.set_flow_run_state(
  File "/usr/local/lib/python3.9/site-packages/prefect/client/orion.py", line 1641, in set_flow_run_state
    response = await self._client.post(
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1848, in post
    return await self.request(
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1533, in request
    return await self.send(request, auth=auth, follow_redirects=follow_redirects)
  File "/usr/local/lib/python3.9/site-packages/prefect/client/base.py", line 229, in send
    response = await self._send_with_retry(
  File "/usr/local/lib/python3.9/site-packages/prefect/client/base.py", line 187, in _send_with_retry
    response = await request()
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1620, in send
    response = await self._send_handling_auth(
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1648, in _send_handling_auth
    response = await self._send_handling_redirects(
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1685, in _send_handling_redirects
    response = await self._send_single_request(request)
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1722, in _send_single_request
    response = await transport.handle_async_request(request)
  File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
    resp = await self._pool.handle_async_request(req)
  File "/usr/local/lib/python3.9/contextlib.py", line 137, in __exit__
    self.gen.throw(typ, value, traceback)
  File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 77, in map_httpcore_exceptions
    raise mapped_exc(message) from exc
httpx.ReadTimeout
--- Orion logging error ---
Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/anyio/streams/tls.py", line 130, in _call_sslobject_method
    result = func(*args)
  File "/usr/local/lib/python3.9/ssl.py", line 889, in read
    v = self._sslobj.read(len)
ssl.SSLWantReadError: The operation did not complete (read) (_ssl.c:2633)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/httpcore/backends/asyncio.py", line 34, in read
    return await self._stream.receive(max_bytes=max_bytes)
  File "/usr/local/lib/python3.9/site-packages/anyio/streams/tls.py", line 195, in receive
    data = await self._call_sslobject_method(self._ssl_object.read, max_bytes)
  File "/usr/local/lib/python3.9/site-packages/anyio/streams/tls.py", line 137, in _call_sslobject_method
    data = await self.transport_stream.receive()
  File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 1265, in receive
    await self._protocol.read_event.wait()
  File "/usr/local/lib/python3.9/asyncio/locks.py", line 226, in wait
    await fut
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/httpcore/_exceptions.py", line 10, in map_exceptions
    yield
  File "/usr/local/lib/python3.9/site-packages/httpcore/backends/asyncio.py", line 36, in read
    return b""
  File "/usr/local/lib/python3.9/site-packages/anyio/_core/_tasks.py", line 118, in __exit__
    raise TimeoutError
TimeoutError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 60, in map_httpcore_exceptions
    yield
  File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
    resp = await self._pool.handle_async_request(req)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection_pool.py", line 253, in handle_async_request
    raise exc
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection_pool.py", line 237, in handle_async_request
    response = await connection.handle_async_request(request)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection.py", line 90, in handle_async_request
    return await self._connection.handle_async_request(request)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 144, in handle_async_request
    raise exc
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 108, in handle_async_request
    await self._send_request_body(request=request, stream_id=stream_id)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 214, in _send_request_body
    max_flow = await self._wait_for_outgoing_flow(request, stream_id)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 386, in _wait_for_outgoing_flow
    await self._receive_events(request)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 281, in _receive_events
    events = await self._read_incoming_data(request)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 341, in _read_incoming_data
    raise exc
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 327, in _read_incoming_data
    data = await self._network_stream.read(self.READ_NUM_BYTES, timeout)
  File "/usr/local/lib/python3.9/site-packages/httpcore/backends/asyncio.py", line 36, in read
    return b""
  File "/usr/local/lib/python3.9/contextlib.py", line 137, in __exit__
    self.gen.throw(typ, value, traceback)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_exceptions.py", line 14, in map_exceptions
    raise to_exc(exc)
httpcore.ReadTimeout

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/prefect/logging/handlers.py", line 151, in send_logs
    await client.create_logs(self._pending_logs)
  File "/usr/local/lib/python3.9/site-packages/prefect/client/orion.py", line 1848, in create_logs
    await self._client.post(f"/logs/", json=serialized_logs)
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1848, in post
    return await self.request(
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1533, in request
    return await self.send(request, auth=auth, follow_redirects=follow_redirects)
  File "/usr/local/lib/python3.9/site-packages/prefect/client/base.py", line 229, in send
    response = await self._send_with_retry(
  File "/usr/local/lib/python3.9/site-packages/prefect/client/base.py", line 187, in _send_with_retry
    response = await request()
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1620, in send
    response = await self._send_handling_auth(
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1648, in _send_handling_auth
    response = await self._send_handling_redirects(
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1685, in _send_handling_redirects
    response = await self._send_single_request(request)
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1722, in _send_single_request
    response = await transport.handle_async_request(request)
  File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
    resp = await self._pool.handle_async_request(req)
  File "/usr/local/lib/python3.9/contextlib.py", line 137, in __exit__
    self.gen.throw(typ, value, traceback)
  File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 77, in map_httpcore_exceptions
    raise mapped_exc(message) from exc
httpx.ReadTimeout
Worker information:
    Approximate queue length: 1
    Pending log batch length: 1
    Pending log batch size: 360
The log worker will attempt to send these logs again in 2.0s
--- Orion logging error ---
Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/anyio/streams/tls.py", line 130, in _call_sslobject_method
    result = func(*args)
  File "/usr/local/lib/python3.9/ssl.py", line 889, in read
    v = self._sslobj.read(len)
ssl.SSLWantReadError: The operation did not complete (read) (_ssl.c:2633)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/httpcore/backends/asyncio.py", line 34, in read
    return await self._stream.receive(max_bytes=max_bytes)
  File "/usr/local/lib/python3.9/site-packages/anyio/streams/tls.py", line 195, in receive
    data = await self._call_sslobject_method(self._ssl_object.read, max_bytes)
  File "/usr/local/lib/python3.9/site-packages/anyio/streams/tls.py", line 137, in _call_sslobject_method
    data = await self.transport_stream.receive()
  File "/usr/local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 1265, in receive
    await self._protocol.read_event.wait()
  File "/usr/local/lib/python3.9/asyncio/locks.py", line 226, in wait
    await fut
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/httpcore/_exceptions.py", line 10, in map_exceptions
    yield
  File "/usr/local/lib/python3.9/site-packages/httpcore/backends/asyncio.py", line 36, in read
    return b""
  File "/usr/local/lib/python3.9/site-packages/anyio/_core/_tasks.py", line 118, in __exit__
    raise TimeoutError
TimeoutError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 60, in map_httpcore_exceptions
    yield
  File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
    resp = await self._pool.handle_async_request(req)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection_pool.py", line 253, in handle_async_request
    raise exc
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection_pool.py", line 237, in handle_async_request
    response = await connection.handle_async_request(request)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/connection.py", line 90, in handle_async_request
    return await self._connection.handle_async_request(request)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 144, in handle_async_request
    raise exc
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 108, in handle_async_request
    await self._send_request_body(request=request, stream_id=stream_id)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 214, in _send_request_body
    max_flow = await self._wait_for_outgoing_flow(request, stream_id)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 386, in _wait_for_outgoing_flow
    await self._receive_events(request)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 281, in _receive_events
    events = await self._read_incoming_data(request)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 341, in _read_incoming_data
    raise exc
  File "/usr/local/lib/python3.9/site-packages/httpcore/_async/http2.py", line 327, in _read_incoming_data
    data = await self._network_stream.read(self.READ_NUM_BYTES, timeout)
  File "/usr/local/lib/python3.9/site-packages/httpcore/backends/asyncio.py", line 36, in read
    return b""
  File "/usr/local/lib/python3.9/contextlib.py", line 137, in __exit__
    self.gen.throw(typ, value, traceback)
  File "/usr/local/lib/python3.9/site-packages/httpcore/_exceptions.py", line 14, in map_exceptions
    raise to_exc(exc)
httpcore.ReadTimeout

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/prefect/logging/handlers.py", line 151, in send_logs
    await client.create_logs(self._pending_logs)
  File "/usr/local/lib/python3.9/site-packages/prefect/client/orion.py", line 1848, in create_logs
    await self._client.post(f"/logs/", json=serialized_logs)
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1848, in post
    return await self.request(
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1533, in request
    return await self.send(request, auth=auth, follow_redirects=follow_redirects)
  File "/usr/local/lib/python3.9/site-packages/prefect/client/base.py", line 229, in send
    response = await self._send_with_retry(
  File "/usr/local/lib/python3.9/site-packages/prefect/client/base.py", line 187, in _send_with_retry
    response = await request()
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1620, in send
    response = await self._send_handling_auth(
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1648, in _send_handling_auth
    response = await self._send_handling_redirects(
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1685, in _send_handling_redirects
    response = await self._send_single_request(request)
  File "/usr/local/lib/python3.9/site-packages/httpx/_client.py", line 1722, in _send_single_request
    response = await transport.handle_async_request(request)
  File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 353, in handle_async_request
    resp = await self._pool.handle_async_request(req)
  File "/usr/local/lib/python3.9/contextlib.py", line 137, in __exit__
    self.gen.throw(typ, value, traceback)
  File "/usr/local/lib/python3.9/site-packages/httpx/_transports/default.py", line 77, in map_httpcore_exceptions
    raise mapped_exc(message) from exc
httpx.ReadTimeout
Worker information:
    Approximate queue length: 0
    Pending log batch length: 2
    Pending log batch size: 720
```


### Versions

```
2.7.10
```


### Additional context

_No response_


Discussion:
Hm we have a size check at https://github.com/PrefectHQ/prefect/blob/main/src/prefect/logging/handlers.py#L354-L359 but checking the size of objects in Python is notoriously not effective so we'll probably need to update that to create JSON directly and measure the length.

Resolving patch:
diff --git a/src/prefect/logging/handlers.py b/src/prefect/logging/handlers.py
--- a/src/prefect/logging/handlers.py
+++ b/src/prefect/logging/handlers.py
@@ -1,4 +1,5 @@
 import atexit
+import json
 import logging
 import queue
 import sys
@@ -6,7 +7,7 @@
 import time
 import traceback
 import warnings
-from typing import Dict, List, Union
+from typing import Any, Dict, List, Tuple, Union
 
 import anyio
 import pendulum
@@ -38,7 +39,7 @@ class OrionLogWorker:
     def __init__(self, profile_context: prefect.context.SettingsContext) -> None:
         self.profile_context = profile_context.copy()
 
-        self._queue: queue.Queue[dict] = queue.Queue()
+        self._queue: queue.Queue[Tuple[Dict[str, Any], int]] = queue.Queue()
 
         self._send_thread = threading.Thread(
             target=self._send_logs_loop,
@@ -135,9 +136,9 @@ async def send_logs(self, exiting: bool = False) -> None:
             # Pull logs from the queue until it is empty or we reach the batch size
             try:
                 while self._pending_size < max_batch_size:
-                    log = self._queue.get_nowait()
+                    log, log_size = self._queue.get_nowait()
                     self._pending_logs.append(log)
-                    self._pending_size += sys.getsizeof(log)
+                    self._pending_size += log_size
 
             except queue.Empty:
                 done = True
@@ -193,12 +194,12 @@ def worker_info(self) -> str:
             f"    Pending log batch size: {self._pending_size}\n"
         )
 
-    def enqueue(self, log: LogCreate):
+    def enqueue(self, log: Dict[str, Any], log_size: int):
         if self._stopped:
             raise RuntimeError(
                 "Logs cannot be enqueued after the Orion log worker is stopped."
             )
-        self._queue.put(log)
+        self._queue.put((log, log_size))
 
     def flush(self, block: bool = False) -> None:
         with self._lock:
@@ -276,7 +277,8 @@ def emit(self, record: logging.LogRecord):
             if not getattr(record, "send_to_orion", True):
                 return  # Do not send records that have opted out
 
-            self.get_worker(profile).enqueue(self.prepare(record, profile.settings))
+            log, log_size = self.prepare(record, profile.settings)
+            self.get_worker(profile).enqueue(log, log_size)
         except Exception:
             self.handleError(record)
 
@@ -302,7 +304,7 @@ def handleError(self, record: logging.LogRecord) -> None:
 
     def prepare(
         self, record: logging.LogRecord, settings: prefect.settings.Settings
-    ) -> LogCreate:
+    ) -> Tuple[Dict[str, Any], int]:
         """
         Convert a `logging.LogRecord` to the Orion `LogCreate` schema and serialize.
 
@@ -351,14 +353,14 @@ def prepare(
             message=self.format(record),
         ).dict(json_compatible=True)
 
-        log_size = sys.getsizeof(log)
+        log_size = self.get_log_size(log)
         if log_size > PREFECT_LOGGING_ORION_MAX_LOG_SIZE.value():
             raise ValueError(
                 f"Log of size {log_size} is greater than the max size of "
                 f"{PREFECT_LOGGING_ORION_MAX_LOG_SIZE.value()}"
             )
 
-        return log
+        return (log, log_size)
 
     def close(self) -> None:
         """
@@ -370,6 +372,9 @@ def close(self) -> None:
 
         return super().close()
 
+    def get_log_size(self, log: Dict[str, Any]) -> int:
+        return len(json.dumps(log).encode())
+
 
 class PrefectConsoleHandler(logging.StreamHandler):
     def __init__(


Test patch:
diff --git a/tests/test_logging.py b/tests/test_logging.py
--- a/tests/test_logging.py
+++ b/tests/test_logging.py
@@ -354,8 +354,9 @@ def test_sends_task_run_log_to_worker(self, logger, mock_log_worker, task_run):
             message="test-task",
         ).dict(json_compatible=True)
         expected["timestamp"] = ANY  # Tested separately
+        log_size = ANY  # Tested separately
 
-        mock_log_worker().enqueue.assert_called_once_with(expected)
+        mock_log_worker().enqueue.assert_called_once_with(expected, log_size)
 
     def test_sends_flow_run_log_to_worker(self, logger, mock_log_worker, flow_run):
         with FlowRunContext.construct(flow_run=flow_run):
@@ -369,8 +370,9 @@ def test_sends_flow_run_log_to_worker(self, logger, mock_log_worker, flow_run):
             message="test-flow",
         ).dict(json_compatible=True)
         expected["timestamp"] = ANY  # Tested separately
+        log_size = ANY  # Tested separately
 
-        mock_log_worker().enqueue.assert_called_once_with(expected)
+        mock_log_worker().enqueue.assert_called_once_with(expected, log_size)
 
     @pytest.mark.parametrize("with_context", [True, False])
     def test_respects_explicit_flow_run_id(
@@ -393,8 +395,9 @@ def test_respects_explicit_flow_run_id(
             message="test-task",
         ).dict(json_compatible=True)
         expected["timestamp"] = ANY  # Tested separately
+        log_size = ANY  # Tested separately
 
-        mock_log_worker().enqueue.assert_called_once_with(expected)
+        mock_log_worker().enqueue.assert_called_once_with(expected, log_size)
 
     @pytest.mark.parametrize("with_context", [True, False])
     def test_respects_explicit_task_run_id(
@@ -418,8 +421,9 @@ def test_respects_explicit_task_run_id(
             message="test-task",
         ).dict(json_compatible=True)
         expected["timestamp"] = ANY  # Tested separately
+        log_size = ANY  # Tested separately
 
-        mock_log_worker().enqueue.assert_called_once_with(expected)
+        mock_log_worker().enqueue.assert_called_once_with(expected, log_size)
 
     def test_does_not_emit_logs_below_level(self, logger, mock_log_worker):
         logger.setLevel(logging.WARNING)
@@ -447,10 +451,10 @@ def test_sets_timestamp_from_record_created_time(
             logger.info("test-flow")
 
         record = handler.emit.call_args[0][0]
-        log_json = mock_log_worker().enqueue.call_args[0][0]
+        log_dict = mock_log_worker().enqueue.call_args[0][0]
 
         assert (
-            log_json["timestamp"] == pendulum.from_timestamp(record.created).isoformat()
+            log_dict["timestamp"] == pendulum.from_timestamp(record.created).isoformat()
         )
 
     def test_sets_timestamp_from_time_if_missing_from_recrod(
@@ -470,9 +474,9 @@ def drop_created_and_emit(emit, record):
         with FlowRunContext.construct(flow_run=flow_run):
             logger.info("test-flow")
 
-        log_json = mock_log_worker().enqueue.call_args[0][0]
+        log_dict = mock_log_worker().enqueue.call_args[0][0]
 
-        assert log_json["timestamp"] == pendulum.from_timestamp(now).isoformat()
+        assert log_dict["timestamp"] == pendulum.from_timestamp(now).isoformat()
 
     def test_does_not_send_logs_that_opt_out(self, logger, mock_log_worker, task_run):
         with TaskRunContext.construct(task_run=task_run):
@@ -613,10 +617,25 @@ async def test_does_not_enqueue_logs_that_are_too_big(
         assert "ValueError" in output.err
         assert "is greater than the max size of 1" in output.err
 
+    def test_handler_knows_how_large_logs_are(self):
+        dict_log = {
+            "name": "prefect.flow_runs",
+            "level": 20,
+            "message": "Finished in state Completed()",
+            "timestamp": "2023-02-08T17:55:52.993831+00:00",
+            "flow_run_id": "47014fb1-9202-4a78-8739-c993d8c24415",
+            "task_run_id": None,
+        }
+
+        log_size = len(json.dumps(dict_log))
+        assert log_size == 211
+        handler = OrionHandler()
+        assert handler.get_log_size(dict_log) == log_size
+
 
 class TestOrionLogWorker:
     @pytest.fixture
-    def log_json(self):
+    def log_dict(self):
         return LogCreate(
             flow_run_id=uuid.uuid4(),
             task_run_id=uuid.uuid4(),
@@ -626,6 +645,10 @@ def log_json(self):
             message="hello",
         ).dict(json_compatible=True)
 
+    @pytest.fixture
+    def log_size(self, log_dict) -> int:
+        return OrionHandler().get_log_size(log_dict)
+
     @pytest.fixture
     def worker(self, get_worker):
         yield get_worker()
@@ -670,26 +693,29 @@ def test_stop_is_idempotent(self, worker):
         worker._stop_event.set.assert_called_once()
         worker._send_thread.join.assert_called_once()
 
-    def test_enqueue(self, log_json, worker):
-        worker.enqueue(log_json)
-        assert worker._queue.get_nowait() == log_json
+    def test_enqueue(self, log_dict, log_size, worker):
+        worker.enqueue(log_dict, log_size)
+        assert worker._queue.get_nowait() == (log_dict, log_size)
 
-    async def test_send_logs_single_record(self, log_json, orion_client, worker):
-        worker.enqueue(log_json)
+    async def test_send_logs_single_record(
+        self, log_dict, log_size, orion_client, worker
+    ):
+        worker.enqueue(log_dict, log_size)
         await worker.send_logs()
         logs = await orion_client.read_logs()
         assert len(logs) == 1
-        assert logs[0].dict(include=log_json.keys(), json_compatible=True) == log_json
+        assert logs[0].dict(include=log_dict.keys(), json_compatible=True) == log_dict
 
-    async def test_send_logs_many_records(self, log_json, orion_client, worker):
+    async def test_send_logs_many_records(self, log_dict, orion_client, worker):
         # Use the read limit as the count since we'd need multiple read calls otherwise
         count = prefect.settings.PREFECT_ORION_API_DEFAULT_LIMIT.value()
-        log_json.pop("message")
+        log_dict.pop("message")
 
         for i in range(count):
-            new_log = log_json.copy()
+            new_log = log_dict.copy()
             new_log["message"] = str(i)
-            worker.enqueue(new_log)
+            new_log_size = OrionHandler().get_log_size(new_log)
+            worker.enqueue(new_log, new_log_size)
         await worker.send_logs()
 
         logs = await orion_client.read_logs()
@@ -697,14 +723,14 @@ async def test_send_logs_many_records(self, log_json, orion_client, worker):
         for log in logs:
             assert (
                 log.dict(
-                    include=log_json.keys(), exclude={"message"}, json_compatible=True
+                    include=log_dict.keys(), exclude={"message"}, json_compatible=True
                 )
-                == log_json
+                == log_dict
             )
         assert len(set(log.message for log in logs)) == count, "Each log is unique"
 
     async def test_send_logs_retries_on_next_call_on_exception(
-        self, log_json, orion_client, monkeypatch, capsys, worker
+        self, log_dict, log_size, orion_client, monkeypatch, capsys, worker
     ):
         create_logs = orion_client.create_logs
         monkeypatch.setattr(
@@ -712,11 +738,11 @@ async def test_send_logs_retries_on_next_call_on_exception(
             MagicMock(side_effect=ValueError("Test")),
         )
 
-        worker.enqueue(log_json)
+        worker.enqueue(log_dict, log_size)
         await worker.send_logs()
 
         # Log moved from queue to pending logs
-        assert worker._pending_logs == [log_json]
+        assert worker._pending_logs == [log_dict]
         with pytest.raises(queue.Empty):
             worker._queue.get_nowait()
 
@@ -732,14 +758,14 @@ async def test_send_logs_retries_on_next_call_on_exception(
 
     @pytest.mark.parametrize("exiting", [True, False])
     async def test_send_logs_writes_exceptions_to_stderr(
-        self, log_json, capsys, monkeypatch, exiting, worker
+        self, log_dict, log_size, capsys, monkeypatch, exiting, worker
     ):
         monkeypatch.setattr(
             "prefect.client.OrionClient.create_logs",
             MagicMock(side_effect=ValueError("Test")),
         )
 
-        worker.enqueue(log_json)
+        worker.enqueue(log_dict, log_size)
         await worker.send_logs(exiting=exiting)
 
         err = capsys.readouterr().err
@@ -750,28 +776,29 @@ async def test_send_logs_writes_exceptions_to_stderr(
         else:
             assert "log worker is stopping and these logs will not be sent" in err
 
-    async def test_send_logs_batches_by_size(self, log_json, monkeypatch, get_worker):
-        test_log_size = sys.getsizeof(log_json)
+    async def test_send_logs_batches_by_size(
+        self, log_dict, log_size, monkeypatch, get_worker
+    ):
         mock_create_logs = AsyncMock()
         monkeypatch.setattr("prefect.client.OrionClient.create_logs", mock_create_logs)
 
         with temporary_settings(
             updates={
-                PREFECT_LOGGING_ORION_BATCH_SIZE: test_log_size + 1,
-                PREFECT_LOGGING_ORION_MAX_LOG_SIZE: test_log_size,
+                PREFECT_LOGGING_ORION_BATCH_SIZE: log_size + 1,
+                PREFECT_LOGGING_ORION_MAX_LOG_SIZE: log_size,
             }
         ):
             worker = get_worker()
-            worker.enqueue(log_json)
-            worker.enqueue(log_json)
-            worker.enqueue(log_json)
+            worker.enqueue(log_dict, log_size)
+            worker.enqueue(log_dict, log_size)
+            worker.enqueue(log_dict, log_size)
             await worker.send_logs()
 
         assert mock_create_logs.call_count == 3
 
     @pytest.mark.flaky(max_runs=3)
     async def test_logs_are_sent_when_started(
-        self, log_json, orion_client, get_worker, monkeypatch
+        self, log_dict, log_size, orion_client, get_worker, monkeypatch
     ):
         event = threading.Event()
         unpatched_create_logs = orion_client.create_logs
@@ -787,9 +814,9 @@ async def create_logs(self, *args, **kwargs):
             updates={PREFECT_LOGGING_ORION_BATCH_INTERVAL: "0.001"}
         ):
             worker = get_worker()
-            worker.enqueue(log_json)
+            worker.enqueue(log_dict, log_size)
             worker.start()
-            worker.enqueue(log_json)
+            worker.enqueue(log_dict, log_size)
 
         # We want to ensure logs are written without the thread being joined
         await asyncio.sleep(0.01)
@@ -821,15 +848,15 @@ def test_flush_event_is_cleared(self, get_worker):
         worker._flush_event.clear.assert_called()
 
     async def test_logs_are_sent_immediately_when_stopped(
-        self, log_json, orion_client, get_worker
+        self, log_dict, log_size, orion_client, get_worker
     ):
         # Set a long interval
         start_time = time.time()
         with temporary_settings(updates={PREFECT_LOGGING_ORION_BATCH_INTERVAL: "10"}):
             worker = get_worker()
-            worker.enqueue(log_json)
+            worker.enqueue(log_dict, log_size)
             worker.start()
-            worker.enqueue(log_json)
+            worker.enqueue(log_dict, log_size)
             worker.stop()
         end_time = time.time()
 
@@ -840,30 +867,30 @@ async def test_logs_are_sent_immediately_when_stopped(
         logs = await orion_client.read_logs()
         assert len(logs) == 2
 
-    async def test_raises_on_enqueue_after_stop(self, worker, log_json):
+    async def test_raises_on_enqueue_after_stop(self, worker, log_dict, log_size):
         worker.start()
         worker.stop()
         with pytest.raises(
             RuntimeError, match="Logs cannot be enqueued after .* is stopped"
         ):
-            worker.enqueue(log_json)
+            worker.enqueue(log_dict, log_size)
 
-    async def test_raises_on_start_after_stop(self, worker, log_json):
+    async def test_raises_on_start_after_stop(self, worker):
         worker.start()
         worker.stop()
         with pytest.raises(RuntimeError, match="cannot be started after stopping"):
             worker.start()
 
     async def test_logs_are_sent_immediately_when_flushed(
-        self, log_json, orion_client, get_worker
+        self, log_dict, log_size, orion_client, get_worker
     ):
         # Set a long interval
         start_time = time.time()
         with temporary_settings(updates={PREFECT_LOGGING_ORION_BATCH_INTERVAL: "10"}):
             worker = get_worker()
-            worker.enqueue(log_json)
+            worker.enqueue(log_dict, log_size)
             worker.start()
-            worker.enqueue(log_json)
+            worker.enqueue(log_dict, log_size)
             worker.flush(block=True)
         end_time = time.time()
 
@@ -875,18 +902,18 @@ async def test_logs_are_sent_immediately_when_flushed(
         assert len(logs) == 2
 
     async def test_logs_can_be_flushed_repeatedly(
-        self, log_json, orion_client, get_worker
+        self, log_dict, log_size, orion_client, get_worker
     ):
         # Set a long interval
         start_time = time.time()
         with temporary_settings(updates={PREFECT_LOGGING_ORION_BATCH_INTERVAL: "10"}):
             worker = get_worker()
-            worker.enqueue(log_json)
+            worker.enqueue(log_dict, log_size)
             worker.start()
-            worker.enqueue(log_json)
+            worker.enqueue(log_dict, log_size)
             worker.flush()
             worker.flush()
-            worker.enqueue(log_json)
+            worker.enqueue(log_dict, log_size)
             worker.flush(block=True)
         end_time = time.time()