Repository: igorbenav/fastcrud
Pull request #62 (2024-04-25 03:36:33)

Add unpaginated versions of multi-row get methods (w/tests)
## Description

Per #59, this PR is to add new methods to `FastCRUD` to do multi-row gets (without and with joins) that return all rows in a single list without pagination.

## Changes

Added new methods `get_multi_all` and `get_multi_joined_all` that are basically copies minus the `offset`, `limit`, and `return_total_count` args as well as without wrapping the list of result rows in a `dict` (since that happens in the existing methods to help with pagination).

## Tests

`test_get_multi.py` and `test_get_multi_joined.py` for both `sqlalchemy` and `sqlmodel` were copied into new files and adjusted to test the new methods (including removing tests that only made sense in the context of pagination).

## Checklist
- [X] I have read the [CONTRIBUTING](CONTRIBUTING.md) document.
- [X] My code follows the code style of this project.
- [X] I have added necessary documentation (if appropriate).
- [X] I have added tests that cover my changes (if applicable).
- [X] All new and existing tests passed.


igorbenav: @slaarti could you please refactor? I think doing it [this way](https://github.com/igorbenav/fastcrud/issues/59#issuecomment-2076287486) would be bettet indeed
slaarti: Okay, sure. Not tonight, might take a couple days, but I'm fine with the alternate approach. 👍
igorbenav: Nice, thanks!
codecov[bot]: ## [Codecov](https://app.codecov.io/gh/igorbenav/fastcrud/pull/62?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Igor+Benav) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 100.00%. Comparing base [(`5e45aee`)](https://app.codecov.io/gh/igorbenav/fastcrud/commit/5e45aee16db9b541d98886472618ec967a3c527b?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Igor+Benav) to head [(`bcbc232`)](https://app.codecov.io/gh/igorbenav/fastcrud/pull/62?dropdown=coverage&src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Igor+Benav).


<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##              main       #62    +/-   ##
==========================================
  Coverage   100.00%   100.00%            
==========================================
  Files           64        68     +4     
  Lines         4282      5062   +780     
==========================================
+ Hits          4282      5062   +780     
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/igorbenav/fastcrud/pull/62?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Igor+Benav).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=Igor+Benav).


Patch:
diff --git a/fastcrud/crud/fast_crud.py b/fastcrud/crud/fast_crud.py
index 617d7ec..f813407 100644
--- a/fastcrud/crud/fast_crud.py
+++ b/fastcrud/crud/fast_crud.py
@@ -585,7 +585,7 @@ class FastCRUD(
         self,
         db: AsyncSession,
         offset: int = 0,
-        limit: int = 100,
+        limit: Optional[int] = 100,
         schema_to_select: Optional[type[BaseModel]] = None,
         sort_columns: Optional[Union[str, list[str]]] = None,
         sort_orders: Optional[Union[str, list[str]]] = None,
@@ -604,7 +604,7 @@ class FastCRUD(
         Args:
             db: The database session to use for the operation.
             offset: Starting index for records to fetch, useful for pagination.
-            limit: Maximum number of records to fetch in one call.
+            limit: Maximum number of records to fetch in one call. Use `None` for "no limit", fetching all matching rows. Note that in order to use `limit=None`, you'll have to provide a custom endpoint to facilitate it, which you should only do if you really seriously want to allow the user to get all the data at once.
             schema_to_select: Optional Pydantic schema for selecting specific columns. Required if `return_as_model` is True.
             sort_columns: Column names to sort the results by.
             sort_orders: Corresponding sort orders ('asc', 'desc') for each column in sort_columns.
@@ -649,7 +649,7 @@ class FastCRUD(
             users = await crud.get_multi(db, 0, 10, is_active=True, sort_columns=['username', 'email'], sort_orders=['asc', 'desc'])
             ```
         """
-        if limit < 0 or offset < 0:
+        if (limit is not None and limit < 0) or offset < 0:
             raise ValueError("Limit and offset must be non-negative.")
 
         stmt = await self.select(
@@ -659,7 +659,11 @@ class FastCRUD(
             **kwargs,
         )
 
-        stmt = stmt.offset(offset).limit(limit)
+        if offset:
+            stmt = stmt.offset(offset)
+        if limit is not None:
+            stmt = stmt.limit(limit)
+
         result = await db.execute(stmt)
         data = [dict(row) for row in result.mappings()]
 
@@ -959,7 +963,7 @@ class FastCRUD(
         join_filters: Optional[dict] = None,
         nest_joins: bool = False,
         offset: int = 0,
-        limit: int = 100,
+        limit: Optional[int] = 100,
         sort_columns: Optional[Union[str, list[str]]] = None,
         sort_orders: Optional[Union[str, list[str]]] = None,
         return_as_model: bool = False,
@@ -988,7 +992,7 @@ class FastCRUD(
             join_filters: Filters applied to the joined model, specified as a dictionary mapping column names to their expected values.
             nest_joins: If True, nested data structures will be returned where joined model data are nested under the join_prefix as a dictionary.
             offset: The offset (number of records to skip) for pagination.
-            limit: The limit (maximum number of records to return) for pagination.
+            limit: Maximum number of records to fetch in one call. Use `None` for "no limit", fetching all matching rows. Note that in order to use `limit=None`, you'll have to provide a custom endpoint to facilitate it, which you should only do if you really seriously want to allow the user to get all the data at once.
             sort_columns: A single column name or a list of column names on which to apply sorting.
             sort_orders: A single sort order ('asc' or 'desc') or a list of sort orders corresponding to the columns in sort_columns. If not provided, defaults to 'asc' for each column.
             return_as_model: If True, converts the fetched data to Pydantic models based on schema_to_select. Defaults to False.
@@ -1205,7 +1209,7 @@ class FastCRUD(
         elif not joins_config and not join_model:
             raise ValueError("You need one of join_model or joins_config.")
 
-        if limit < 0 or offset < 0:
+        if (limit is not None and limit < 0) or offset < 0:
             raise ValueError("Limit and offset must be non-negative.")
 
         primary_select = _extract_matching_columns_from_schema(
@@ -1260,7 +1264,10 @@ class FastCRUD(
         if sort_columns:
             stmt = self._apply_sorting(stmt, sort_columns, sort_orders)
 
-        stmt = stmt.offset(offset).limit(limit)
+        if offset:
+            stmt = stmt.offset(offset)
+        if limit is not None:
+            stmt = stmt.limit(limit)
 
         result = await db.execute(stmt)
         data: list[Union[dict, BaseModel]] = []


Test patch:
diff --git a/tests/sqlalchemy/crud/test_get_multi.py b/tests/sqlalchemy/crud/test_get_multi.py
index 4334763..9b5a837 100644
--- a/tests/sqlalchemy/crud/test_get_multi.py
+++ b/tests/sqlalchemy/crud/test_get_multi.py
@@ -48,6 +48,24 @@ async def test_get_multi_pagination(async_session, test_model, test_data):
     assert len(result_page_2["data"]) == min(5, max(0, total_count - 5))
 
 
+@pytest.mark.asyncio
+async def test_get_multi_unpaginated(async_session, test_model, test_data):
+    for item in test_data:
+        record = test_model(**item)
+        async_session.add(record)
+    await async_session.commit()
+
+    total_count_query = await async_session.execute(
+        select(func.count()).select_from(test_model)
+    )
+    total_count = total_count_query.scalar()
+
+    crud = FastCRUD(test_model)
+    results = await crud.get_multi(async_session, offset=0, limit=None)
+
+    assert len(results["data"]) == total_count
+
+
 @pytest.mark.asyncio
 async def test_get_multi_sorting(async_session, test_model, test_data):
     for item in test_data:
diff --git a/tests/sqlalchemy/crud/test_get_multi_joined.py b/tests/sqlalchemy/crud/test_get_multi_joined.py
index d8d0317..3dfe729 100644
--- a/tests/sqlalchemy/crud/test_get_multi_joined.py
+++ b/tests/sqlalchemy/crud/test_get_multi_joined.py
@@ -55,6 +55,32 @@ async def test_get_multi_joined_basic(async_session, test_data, test_data_tier):
     assert all("tier_name" in item for item in result["data"])
 
 
+@pytest.mark.asyncio
+async def test_get_multi_joined_unpaginated(async_session, test_data, test_data_tier):
+    for tier_item in test_data_tier:
+        async_session.add(TierModel(**tier_item))
+    await async_session.commit()
+
+    for user_item in test_data:
+        async_session.add(ModelTest(**user_item))
+    await async_session.commit()
+
+    crud = FastCRUD(ModelTest)
+    result = await crud.get_multi_joined(
+        db=async_session,
+        join_model=TierModel,
+        join_prefix="tier_",
+        schema_to_select=CreateSchemaTest,
+        join_schema_to_select=TierSchemaTest,
+        offset=0,
+        limit=None,
+    )
+
+    assert len(result["data"]) == len(test_data)
+    assert result["total_count"] == len(test_data)
+    assert all("tier_name" in item for item in result["data"])
+
+
 @pytest.mark.asyncio
 async def test_get_multi_joined_sorting(async_session, test_data, test_data_tier):
     for tier_item in test_data_tier:
diff --git a/tests/sqlmodel/crud/test_get_multi.py b/tests/sqlmodel/crud/test_get_multi.py
index 4334763..9b5a837 100644
--- a/tests/sqlmodel/crud/test_get_multi.py
+++ b/tests/sqlmodel/crud/test_get_multi.py
@@ -48,6 +48,24 @@ async def test_get_multi_pagination(async_session, test_model, test_data):
     assert len(result_page_2["data"]) == min(5, max(0, total_count - 5))
 
 
+@pytest.mark.asyncio
+async def test_get_multi_unpaginated(async_session, test_model, test_data):
+    for item in test_data:
+        record = test_model(**item)
+        async_session.add(record)
+    await async_session.commit()
+
+    total_count_query = await async_session.execute(
+        select(func.count()).select_from(test_model)
+    )
+    total_count = total_count_query.scalar()
+
+    crud = FastCRUD(test_model)
+    results = await crud.get_multi(async_session, offset=0, limit=None)
+
+    assert len(results["data"]) == total_count
+
+
 @pytest.mark.asyncio
 async def test_get_multi_sorting(async_session, test_model, test_data):
     for item in test_data:
diff --git a/tests/sqlmodel/crud/test_get_multi_joined.py b/tests/sqlmodel/crud/test_get_multi_joined.py
index ea21d0b..d480721 100644
--- a/tests/sqlmodel/crud/test_get_multi_joined.py
+++ b/tests/sqlmodel/crud/test_get_multi_joined.py
@@ -55,6 +55,32 @@ async def test_get_multi_joined_basic(async_session, test_data, test_data_tier):
     assert all("tier_name" in item for item in result["data"])
 
 
+@pytest.mark.asyncio
+async def test_get_multi_joined_unpaginated(async_session, test_data, test_data_tier):
+    for tier_item in test_data_tier:
+        async_session.add(TierModel(**tier_item))
+    await async_session.commit()
+
+    for user_item in test_data:
+        async_session.add(ModelTest(**user_item))
+    await async_session.commit()
+
+    crud = FastCRUD(ModelTest)
+    result = await crud.get_multi_joined(
+        db=async_session,
+        join_model=TierModel,
+        join_prefix="tier_",
+        schema_to_select=CreateSchemaTest,
+        join_schema_to_select=TierSchemaTest,
+        offset=0,
+        limit=None,
+    )
+
+    assert len(result["data"]) == len(test_data)
+    assert result["total_count"] == len(test_data)
+    assert all("tier_name" in item for item in result["data"])
+
+
 @pytest.mark.asyncio
 async def test_get_multi_joined_sorting(async_session, test_data, test_data_tier):
     for tier_item in test_data_tier: