Repository: apify/crawlee-python
File: src/crawlee/_utils/system.py (Python)

Code before:
    """Retrieve the current memory usage of the process and its children.

    It utilizes the `psutil` library.
    """
    logger.debug('Calling get_memory_info()...')
    current_process = psutil.Process(os.getpid())

    # Retrieve the Resident Set Size (RSS) of the current process. RSS is the portion of memory
    # occupied by a process that is held in RAM.
    # Use RSS as a conservative estimate that can be overestimating due to including shared memory as opposed to USS.
    current_size_bytes = int(current_process.memory_full_info().rss)

    # Sum memory usage by all children processes, try to exclude shared memory from the sum if allowed by OS.
    for child in current_process.children(recursive=True):
        # Ignore any NoSuchProcess exception that might occur if a child process ends before we retrieve
        # its memory usage.
        with suppress(psutil.NoSuchProcess):
            # In the case of children try to estimate memory usage from USS to avoid overestimation due to shared memory
            current_size_bytes += _get_used_memory(child.memory_full_info())

    total_size_bytes = psutil.virtual_memory().total

    return MemoryInfo(
        total_size=ByteSize(total_size_bytes),
        current_size=ByteSize(current_size_bytes),
    )


def _get_used_memory(memory_full_info: Any) -> int:
    """Get the most suitable available used memory metric.

    `Unique Set Size (USS)` is the memory which is unique to a process and which would be freed if the process was
        terminated right now. It should be available on Linux, macOS, Windows.
    `Resident Set Size (RSS)` is the non-swapped physical memory a process has used; it includes shared memory. It
        should be available everywhere.
    """
    try:
        # Overwhelming majority use-case (Linux, macOS, Windows)
        return int(memory_full_info.uss)
    except AttributeError:
        # Very rare use-case
        # Memory usage estimation can overestimate memory usage due to shared memory inclusion.
        return int(memory_full_info.rss)


Review comment (suggestion):
This internal type hint does not seem to be available. The actual type is dependent on the OS as well.

Code after:
    """Retrieve the current memory usage of the process and its children.

    It utilizes the `psutil` library.
    """
    logger.debug('Calling get_memory_info()...')
    current_process = psutil.Process(os.getpid())

    # Retrieve estimated memory usage of the current process.
    current_size_bytes = int(_get_used_memory(current_process.memory_full_info()))

    # Sum memory usage by all children processes, try to exclude shared memory from the sum if allowed by OS.
    for child in current_process.children(recursive=True):
        # Ignore any NoSuchProcess exception that might occur if a child process ends before we retrieve
        # its memory usage.
        with suppress(psutil.NoSuchProcess):
            current_size_bytes += _get_used_memory(child.memory_full_info())

    total_size_bytes = psutil.virtual_memory().total

    return MemoryInfo(
        total_size=ByteSize(total_size_bytes),
        current_size=ByteSize(current_size_bytes),
    )


def _get_used_memory(memory_full_info: Any) -> int:
    """Get the most suitable available used memory metric.

    `Proportional Set Size (PSS)`, is the amount of own memory and memory shared with other processes, accounted in a
    way that the shared amount is divided evenly between the processes that share it. Available on Linux. Suitable for
    avoiding overestimation by counting the same shared memory used by children processes multiple times.

    `Resident Set Size (RSS)` is the non-swapped physical memory a process has used; it includes shared memory. It
    should be available everywhere.
    """
    try:
        # Linux
        return int(memory_full_info.pss)
    except AttributeError:
        return int(memory_full_info.rss)