Repository: edgedb/edgedb

Issue:
Decide on types of array indexing
As noted in #4403, Postgres supports only 32 bit integers and does an implicit cast from `bigint` into `integer` before applying `[]`.

Our `std::[]`, on the other hand, uses `bigint`.

Do we inherit this limitation, or try to work around it?

My opinion is that the limit is reasonable, with `bytea`, this allows up to 2GB. EdgeDB is (probably) not meant for such storage. Similarly, if arrays have more than 2.147.483.648 elements, the user is doing something wrong.

If we decide to inherit, we should:
- [x] correct `std::[]` for arrays, text, bytes and 
- [x] correct `std::[l:u]` (slicing),
- [ ] add some kind of warning in the docs,
- [x] correctly report index out of range errors.



Discussion:
As discussed, there is a distinction between the type in EdgeQL and the type we use in stdlib in Postgres.

Because Postgres does not even support arrays with more than 2^32 elements, we also cannot. But in EdgeQL the type for indexing can remain int64, which we then cast in the implementation.

The important detail is that in EdgeQL, indexing uses int64 and when passing an int that is greater than 2^31, the result will be an index-out-of-bounds error, not an implicit-cast-failed. Currently, this is not the case:

```
edgedb> select [1, 2, 3][15];
edgedb error: InvalidValueError: array index 15 is out of bounds
edgedb> select [1, 2, 3][156468765454544];
edgedb error: NumericOutOfRangeError: std::int32 out of range
```


Resolving patch:
diff --git a/edb/buildmeta.py b/edb/buildmeta.py
--- a/edb/buildmeta.py
+++ b/edb/buildmeta.py
@@ -44,7 +44,7 @@
 
 
 # Increment this whenever the database layout or stdlib changes.
-EDGEDB_CATALOG_VERSION = 2022_09_20_00_00
+EDGEDB_CATALOG_VERSION = 2022_09_29_00_00
 EDGEDB_MAJOR_VERSION = 3
 
 
diff --git a/edb/pgsql/compiler/expr.py b/edb/pgsql/compiler/expr.py
--- a/edb/pgsql/compiler/expr.py
+++ b/edb/pgsql/compiler/expr.py
@@ -286,24 +286,15 @@ def compile_SliceIndirection(
         subctx.expr_exposed = False
         subj = dispatch.compile(expr.expr, ctx=subctx)
 
-        # Postgres arrays use 32bit indexes, so EdgeDB inherits this limitation
-        def compile_and_cast_int(expr: irast.Base) -> pgast.BaseExpr:
-            pg_expr = dispatch.compile(expr, ctx=subctx)
-            return pgast.TypeCast(
-                arg=pg_expr, type_name=pgast.TypeName(name=("integer",))
-            )
-
         if expr.start is None:
             start: pgast.BaseExpr = pgast.LiteralExpr(expr="0")
         else:
-            start = compile_and_cast_int(expr.start)
+            start = dispatch.compile(expr.start, ctx=subctx)
 
         if expr.stop is None:
-            # Max index in EdgeQL is 2^32-2 because during conversion
-            # to Postgres index, we add 1 to the stop and then subtract one.
-            stop: pgast.BaseExpr = pgast.LiteralExpr(expr=str(2**31 - 2))
+            stop: pgast.BaseExpr = pgast.LiteralExpr(expr=str(2**31 - 1))
         else:
-            stop = compile_and_cast_int(expr.stop)
+            stop = dispatch.compile(expr.stop, ctx=subctx)
 
         typ = expr.expr.typeref
         inline_array_slicing = irtyputils.is_array(typ) and any(
@@ -326,22 +317,36 @@ def _inline_array_slicing(
         indirection=[
             pgast.Slice(
                 lidx=pgast.FuncCall(
-                    name=("edgedb", "_normalize_array_index"),
+                    name=("edgedb", "_int_bound"),
                     args=[
-                        start,
-                        pgast.FuncCall(name=("cardinality",), args=[subj]),
+                        pgast.FuncCall(
+                            name=("edgedb", "_normalize_array_index"),
+                            args=[
+                                start,
+                                pgast.FuncCall(
+                                    name=("cardinality",), args=[subj]
+                                ),
+                            ],
+                        )
                     ],
                 ),
-                ridx=astutils.new_binop(
-                    lexpr=pgast.FuncCall(
-                        name=("edgedb", "_normalize_array_index"),
-                        args=[
-                            stop,
-                            pgast.FuncCall(name=("cardinality",), args=[subj]),
-                        ],
-                    ),
-                    op="-",
-                    rexpr=pgast.LiteralExpr(expr="1"),
+                ridx=pgast.FuncCall(
+                    name=("edgedb", "_int_bound"),
+                    args=[
+                        astutils.new_binop(
+                            lexpr=pgast.FuncCall(
+                                name=("edgedb", "_normalize_array_index"),
+                                args=[
+                                    stop,
+                                    pgast.FuncCall(
+                                        name=("cardinality",), args=[subj]
+                                    ),
+                                ],
+                            ),
+                            op="-",
+                            rexpr=pgast.LiteralExpr(expr="1"),
+                        ),
+                    ],
                 ),
             )
         ],
diff --git a/edb/pgsql/metaschema.py b/edb/pgsql/metaschema.py
--- a/edb/pgsql/metaschema.py
+++ b/edb/pgsql/metaschema.py
@@ -1904,31 +1904,79 @@ def __init__(self) -> None:
 
 class NormalizeArrayIndexFunction(dbops.Function):
     """Convert an EdgeQL index to SQL index."""
+
     text = '''
-        SELECT (
+        SELECT
             CASE WHEN index < 0 THEN
-                length + index + 1
+                length::bigint + index + 1
             ELSE
                 index + 1
             END
-        )::int
     '''
 
     def __init__(self) -> None:
         super().__init__(
             name=('edgedb', '_normalize_array_index'),
             args=[('index', ('bigint',)), ('length', ('int',))],
-            returns=('int',),
+            returns=('bigint',),
             volatility='immutable',
             strict=True,
+            text=self.text,
+        )
+
+
+class IntOrNullFunction(dbops.Function):
+    """
+    Convert bigint to int. If it does not fit, return NULL.
+    """
+
+    text = """
+        SELECT
+            CASE WHEN val <= 2147483647 AND val >= -2147483648 THEN
+                val
+            ELSE
+                NULL
+            END
+    """
+
+    def __init__(self) -> None:
+        super().__init__(
+            name=("edgedb", "_int_or_null"),
+            args=[("val", ("bigint",))],
+            returns=("int",),
+            volatility="immutable",
+            strict=True,
+            text=self.text,
+        )
+
+
+class IntBoundFunction(dbops.Function):
+    """
+    Cast bigint to int. If it does not fit, return int's min or max value.
+    """
+
+    text = """
+        SELECT GREATEST(-2147483648, LEAST(2147483647, val))::integer
+    """
+
+    def __init__(self) -> None:
+        super().__init__(
+            name=("edgedb", "_int_bound"),
+            args=[("val", ("bigint",))],
+            returns=("int",),
+            volatility="immutable",
+            strict=True,
             text=self.text)
 
 
 class ArrayIndexWithBoundsFunction(dbops.Function):
     """Get an array element or raise an out-of-bounds exception."""
+
     text = '''
         SELECT edgedb.raise_on_null(
-            val[edgedb._normalize_array_index(index, array_upper(val, 1))],
+            val[edgedb._int_or_null(
+                edgedb._normalize_array_index(index, array_upper(val, 1))
+            )],
             'array_subscript_error',
             msg => 'array index ' || index::text || ' is out of bounds',
             detail => detail
@@ -1951,10 +1999,19 @@ def __init__(self) -> None:
 class ArraySliceFunction(dbops.Function):
     """Get an array slice."""
 
+    # This function is also inlined in expr.py#_inline_array_slicing.
+
+    # Known bug: if array has 2G elements and both bounds are overflowing,
+    # this will return last element instead of an empty array.
     text = """
-        SELECT
-            val[edgedb._normalize_array_index(start, cardinality(val)):
-                edgedb._normalize_array_index(stop, cardinality(val)) - 1]
+        SELECT val[
+            edgedb._int_bound(
+                edgedb._normalize_array_index(start, cardinality(val))
+            ):
+            edgedb._int_bound(
+                edgedb._normalize_array_index(stop, cardinality(val)) - 1
+            )
+        ]
     """
 
     def __init__(self) -> None:
@@ -1962,8 +2019,8 @@ def __init__(self) -> None:
             name=("edgedb", "_slice"),
             args=[
                 ("val", ("anyarray",)),
-                ("start", ("int",)),
-                ("stop", ("int",)),
+                ("start", ("bigint",)),
+                ("stop", ("bigint",)),
             ],
             returns=("anyarray",),
             volatility="immutable",
@@ -1973,17 +2030,23 @@ def __init__(self) -> None:
 
 class StringIndexWithBoundsFunction(dbops.Function):
     """Get a string character or raise an out-of-bounds exception."""
+
     text = '''
         SELECT edgedb.raise_on_empty(
-            substr(
-                "val",
-                edgedb._normalize_array_index("index", char_length("val")),
-                1
-            ),
+            CASE WHEN pg_index IS NULL THEN
+                ''
+            ELSE
+                substr("val", pg_index, 1)
+            END,
             'invalid_parameter_value',
             "typename" || ' index ' || "index"::text || ' is out of bounds',
             "detail"
         )
+        FROM (
+            SELECT edgedb._int_or_null(
+                edgedb._normalize_array_index("index", char_length("val"))
+            ) as pg_index
+        ) t
     '''
 
     def __init__(self) -> None:
@@ -2005,17 +2068,23 @@ def __init__(self) -> None:
 
 class BytesIndexWithBoundsFunction(dbops.Function):
     """Get a bytes character or raise an out-of-bounds exception."""
+
     text = '''
         SELECT edgedb.raise_on_empty(
-            substr(
-                "val",
-                edgedb._normalize_array_index("index", length("val")),
-                1
-            ),
+            CASE WHEN pg_index IS NULL THEN
+                ''::bytea
+            ELSE
+                substr("val", pg_index, 1)
+            END,
             'invalid_parameter_value',
             'byte string index ' || "index"::text || ' is out of bounds',
             "detail"
         )
+        FROM (
+            SELECT edgedb._int_or_null(
+                edgedb._normalize_array_index("index", length("val"))
+            ) as pg_index
+        ) t
     '''
 
     def __init__(self) -> None:
@@ -2099,13 +2168,17 @@ class StringSliceImplFunction(dbops.Function):
         SELECT
             edgedb._substr(
                 val,
-                edgedb._normalize_array_index(
-                    start, edgedb._length(val)),
-                edgedb._normalize_array_index(
-                    stop, edgedb._length(val)) -
-                edgedb._normalize_array_index(
-                    start, edgedb._length(val))
+                pg_start,
+                pg_end - pg_start
             )
+        FROM (SELECT
+            GREATEST(0, edgedb._int_bound(edgedb._normalize_array_index(
+                start, edgedb._length(val)
+            ))) as pg_start,
+            GREATEST(0, edgedb._int_bound(edgedb._normalize_array_index(
+                stop, edgedb._length(val)
+            ))) as pg_end
+        ) t
     """
 
     def __init__(self) -> None:
@@ -2113,8 +2186,8 @@ def __init__(self) -> None:
             name=("edgedb", "_str_slice"),
             args=[
                 ("val", ("anyelement",)),
-                ("start", ("int",)),
-                ("stop", ("int",)),
+                ("start", ("bigint",)),
+                ("stop", ("bigint",)),
             ],
             returns=("anyelement",),
             volatility="immutable",
@@ -2132,7 +2205,9 @@ def __init__(self) -> None:
         super().__init__(
             name=('edgedb', '_slice'),
             args=[
-                ('val', ('text',)), ('start', ('int',)), ('stop', ('int',))
+                ('val', ('text',)),
+                ('start', ('bigint',)),
+                ('stop', ('bigint',)),
             ],
             returns=('text',),
             volatility='immutable',
@@ -2149,7 +2224,9 @@ def __init__(self) -> None:
         super().__init__(
             name=('edgedb', '_slice'),
             args=[
-                ('val', ('bytea',)), ('start', ('int',)), ('stop', ('int',))
+                ('val', ('bytea',)),
+                ('start', ('bigint',)),
+                ('stop', ('bigint',)),
             ],
             returns=('bytea',),
             volatility='immutable',
@@ -2217,6 +2294,7 @@ def __init__(self) -> None:
 
 class JSONIndexByIntFunction(dbops.Function):
     """Get a JSON element by int index or raise an exception."""
+
     text = r'''
         SELECT
             CASE jsonb_typeof(val)
@@ -2233,7 +2311,7 @@ class JSONIndexByIntFunction(dbops.Function):
             )
             WHEN 'array' THEN (
                 edgedb.raise_on_null(
-                    val -> index::int,
+                    val -> edgedb._int_or_null(index),
                     'invalid_parameter_value',
                     msg => 'JSON index ' || index::text || ' is out of bounds',
                     detail => detail
@@ -2319,8 +2397,8 @@ def __init__(self) -> None:
             name=("edgedb", "_slice"),
             args=[
                 ("val", ("jsonb",)),
-                ("start", ("int",)),
-                ("stop", ("int",)),
+                ("start", ("bigint",)),
+                ("stop", ("bigint",)),
             ],
             returns=("jsonb",),
             # Same volatility as to_jsonb (stable)
@@ -4064,6 +4142,8 @@ async def bootstrap(
         dbops.CreateFunction(GetTableDescendantsFunction()),
         dbops.CreateFunction(ParseTriggerConditionFunction()),
         dbops.CreateFunction(NormalizeArrayIndexFunction()),
+        dbops.CreateFunction(IntOrNullFunction()),
+        dbops.CreateFunction(IntBoundFunction()),
         dbops.CreateFunction(ArrayIndexWithBoundsFunction()),
         dbops.CreateFunction(ArraySliceFunction()),
         dbops.CreateFunction(StringIndexWithBoundsFunction()),


Test patch:
diff --git a/tests/test_edgeql_select.py b/tests/test_edgeql_select.py
--- a/tests/test_edgeql_select.py
+++ b/tests/test_edgeql_select.py
@@ -5304,6 +5304,155 @@ async def test_edgeql_select_slice_04(self):
             variables=(None,),
         )
 
+    async def test_edgeql_select_bigint_index_01(self):
+
+        big_pos = str(2**40)
+        big_neg = str(-(2**40))
+
+        async with self.assertRaisesRegexTx(
+            edgedb.InvalidValueError,
+            rf"array index {big_pos} is out of bounds",
+        ):
+            await self.con.query(f"select [1, 2, 3][{big_pos}];")
+
+        async with self.assertRaisesRegexTx(
+            edgedb.InvalidValueError,
+            rf"array index {big_neg} is out of bounds",
+        ):
+            await self.con.query(f"select [1, 2, 3][{big_neg}];")
+
+        await self.assert_query_result(
+            f"""select [1, 2, 3][0:{big_pos}];""",
+            [[1, 2, 3]],
+        )
+
+        await self.assert_query_result(
+            f"""select [1, 2, 3][0:{big_neg}];""",
+            [[]],
+        )
+
+        await self.assert_query_result(
+            f"""select [1, 2, 3][{big_neg}:{big_pos}];""",
+            [[1, 2, 3]],
+        )
+
+        await self.assert_query_result(
+            f"""select [1, 2, 3][{big_pos}:{big_neg}];""",
+            [[]],
+        )
+
+        await self.assert_query_result(
+            f"""select [1, 2, 3][{big_neg}:{big_neg}];""",
+            [[]],
+        )
+
+        await self.assert_query_result(
+            f"""select [1, 2, 3][{big_pos}:{big_pos}];""",
+            [[]],
+        )
+
+    async def test_edgeql_select_bigint_index_02(self):
+
+        big_pos = str(2**40)
+        big_neg = str(-(2**40))
+
+        async with self.assertRaisesRegexTx(
+            edgedb.InvalidValueError,
+            rf"string index {big_pos} is out of bounds",
+        ):
+            await self.con.query(f'select "Hello world!"[{big_pos}];')
+
+        async with self.assertRaisesRegexTx(
+            edgedb.InvalidValueError,
+            rf"string index {big_neg} is out of bounds",
+        ):
+            await self.con.query(f'select "Hello world!"[{big_neg}];')
+
+        await self.assert_query_result(
+            f"""select "Hello world!"[6:{big_pos}];""",
+            ["world!"],
+        )
+
+        await self.assert_query_result(
+            f"""select "Hello world!"[6:{big_neg}];""",
+            [""],
+        )
+
+        await self.assert_query_result(
+            f"""select "Hello world!"[{big_neg}:{big_pos}];""",
+            ["Hello world!"],
+        )
+
+        await self.assert_query_result(
+            f"""select "Hello world!"[{big_pos}:{big_neg}];""",
+            [""],
+        )
+
+        await self.assert_query_result(
+            f"""select "Hello world!"[{big_neg}:{big_neg}];""",
+            [""],
+        )
+
+        await self.assert_query_result(
+            f"""select "Hello world!"[{big_pos}:{big_pos}];""",
+            [""],
+        )
+
+    async def test_edgeql_select_bigint_index_03(self):
+
+        big_pos = str(2**40)
+        big_neg = str(-(2**40))
+
+        async with self.assertRaisesRegexTx(
+            edgedb.InvalidValueError, rf"JSON index {big_pos} is out of bounds"
+        ):
+            await self.con.query(f'select to_json("[1, 2, 3]")[{big_pos}];')
+
+        async with self.assertRaisesRegexTx(
+            edgedb.InvalidValueError, rf"JSON index {big_neg} is out of bounds"
+        ):
+            await self.con.query(f'select to_json("[1, 2, 3]")[{big_neg}];')
+
+        await self.assert_query_result(
+            f"""select to_json("[1, 2, 3]")[1:{big_pos}];""",
+            # JSON:
+            [[2, 3]],
+            # Binary:
+            ['[2, 3]'],
+        )
+
+        await self.assert_query_result(
+            f"""select to_json("[1, 2, 3]")[1:{big_neg}];""",
+            [[]],
+            ['[]'],
+        )
+
+        await self.assert_query_result(
+            f"""select to_json("[1, 2, 3]")[{big_neg}:{big_pos}];""",
+            # JSON:
+            [[1, 2, 3]],
+            # Binary:
+            ['[1, 2, 3]'],
+        )
+
+        await self.assert_query_result(
+            f"""select to_json("[1, 2, 3]")[{big_pos}:{big_neg}];""",
+            [[]],
+            ['[]'],
+        )
+
+        await self.assert_query_result(
+            f"""select to_json("[1, 2, 3]")[{big_neg}:{big_neg}];""",
+            [[]],
+            ['[]'],
+        )
+
+        await self.assert_query_result(
+            f"""select to_json("[1, 2, 3]")[{big_pos}:{big_pos}];""",
+            [[]],
+            ['[]'],
+        )
+
     async def test_edgeql_select_tuple_01(self):
         await self.assert_query_result(
             r"""