Repository: sagemath/sage

Issue:
guessing k-regular sequences
<div id="comment:0"></div>

Code for guessing the parameters of k-regular sequences from the first few entries of a list/sequence/etc.

See also Meta ticket #21202.

Depends on #21319

CC:  @rwst

Component: **combinatorics**

Author: **Daniel Krenn**

Branch/Commit: **[u/cheuberg/sequences/k-regular-guess](https://github.com/sagemath/sagetrac-mirror/tree/u/cheuberg/sequences/k-regular-guess) @ [`ae06d17`](https://github.com/sagemath/sagetrac-mirror/commit/ae06d17a726de124938b43736d68571eca60000a)**

Reviewer: **Clemens Heuberger**

_Issue created by migration from https://trac.sagemath.org/ticket/21204_




Discussion:

Resolving patch:
diff --git a/src/sage/combinat/k_regular_sequence.py b/src/sage/combinat/k_regular_sequence.py
--- a/src/sage/combinat/k_regular_sequence.py
+++ b/src/sage/combinat/k_regular_sequence.py
@@ -101,6 +101,69 @@
 from sage.misc.cachefunc import cached_function, cached_method
 
 
+def pad_right(T, length, zero=0):
+    r"""
+    Pad ``T`` to the right by using ``zero`` to have
+    at least the given ``length``.
+
+    INPUT:
+
+    - ``T`` -- A tuple, list or other iterable
+
+    - ``length`` -- a nonnegative integer
+
+    - ``zero`` -- (default: ``0``) the elements to pad with
+
+    OUTPUT:
+
+    An object of the same type as ``T``
+
+    EXAMPLES::
+
+        sage: from sage.combinat.k_regular_sequence import pad_right
+        sage: pad_right((1, 2, 3), 10)
+        (1, 2, 3, 0, 0, 0, 0, 0, 0, 0)
+        sage: pad_right((1, 2, 3), 2)
+        (1, 2, 3)
+        sage: pad_right([(1, 2), (3, 4)], 4, (0, 0))
+        [(1, 2), (3, 4), (0, 0), (0, 0)]
+
+    TESTS::
+
+        sage: pad_right([1, 2, 3], 10)
+        [1, 2, 3, 0, 0, 0, 0, 0, 0, 0]
+    """
+    return T + type(T)(zero for _ in range(length - len(T)))
+
+
+def value(D, k):
+    r"""
+    Return the value of the expansion with digits `D` in base `k`, i.e.
+
+    .. MATH::
+
+        \sum_{0\leq j < \operatorname{len}D} D[j] k^j.
+
+    INPUT:
+
+    - ``D`` -- a tuple or other iterable
+
+    - ``k`` -- the base
+
+    OUTPUT:
+
+    An element in the common parent of the base `k` and of the entries
+    of `D`
+
+    EXAMPLES::
+
+        sage: from sage.combinat.k_regular_sequence import value
+        sage: value(42.digits(7), 7)
+        42
+    """
+    return sum(d * k**j for j, d in enumerate(D))
+
+
 class kRegularSequence(RecognizableSeries):
     def __init__(self, parent, mu, left=None, right=None):
         r"""
@@ -175,7 +238,7 @@ def _repr_(self):
             preview=10)
 
     @cached_method
-    def __getitem__(self, n, **kwds):
+    def coefficient_of_n(self, n, **kwds):
         r"""
         Return the `n`-th entry of this sequence.
 
@@ -193,6 +256,11 @@ def __getitem__(self, n, **kwds):
             sage: S[7]
             3
 
+        This is equivalent to::
+
+            sage: S.coefficient_of_n(7)
+            3
+
         TESTS::
 
             sage: S[-1]
@@ -216,6 +284,8 @@ def __getitem__(self, n, **kwds):
         """
         return self.coefficient_of_word(self.parent()._n_to_index_(n), **kwds)
 
+    __getitem__ = coefficient_of_n
+
     def __iter__(self):
         r"""
         Return an iterator over the coefficients of this sequence.
@@ -458,7 +528,7 @@ def subsequence(self, a, b):
 
         zero_M = self.mu[0].parent().zero()
         zero_R = self.right.parent().zero()
-        # Let v(n) = self.__getitem__(n, multiply_left=False)
+        # Let v(n) = self.coefficient_of_n(n, multiply_left=False)
         rule = {}
         # We will construct `kernel` and `rule` in such a way that for all
         # c in `kernel`,
@@ -494,7 +564,7 @@ def matrix_row(r, c):
                 b.get(c, 0) * self.left
                 for c in kernel)),
             vector(chain.from_iterable(
-                (self.__getitem__(c, multiply_left=False) if c >= 0 else zero_R)
+                (self.coefficient_of_n(c, multiply_left=False) if c >= 0 else zero_R)
                 for c in kernel)))
 
         return result
@@ -918,6 +988,424 @@ def _n_to_index_(self, n):
         except OverflowError:
             raise ValueError('value {} of index is negative'.format(n)) from None
 
+    def guess(self, f, n_verify=100, max_exponent=10, sequence=None):
+        r"""
+        Guess a `k`-regular sequence whose first terms coincide with `(f(n))_{n\geq0}`.
+
+        INPUT:
+
+        - ``f`` -- a function (callable) which determines the sequence.
+          It takes nonnegative integers as an input
+
+        - ``n_verify`` -- (default: ``100``) a positive integer. The resulting
+          `k`-regular sequence coincides with `f` on the first ``n_verify``
+          terms.
+
+        - ``max_exponent`` -- (default: ``10``) a positive integer specifying
+          the maximum exponent of `k` which is tried when guessing the sequence,
+          i.e., relations between `f(k^t n+r)` are used for
+          `0\le t\le \mathtt{max\_exponent}` and `0\le r < k^j`
+
+        - ``sequence`` -- (default: ``None``) a `k`-regular sequence used
+          for bootstrapping the guessing by adding information of the
+          linear representation of ``sequence`` to the guessed representation
+
+        OUTPUT:
+
+        A :class:`kRegularSequence`
+
+        ALGORITHM:
+
+        For the purposes of this description, the right vector valued sequence
+        associated with a regular sequence consists of the
+        corresponding matrix product multiplied by the right vector,
+        but without the left vector of the regular sequence.
+
+        The algorithm maintains a right vector valued sequence consisting
+        of the right vector valued sequence of the argument ``sequence``
+        (replaced by an empty tuple if ``sequence`` is ``None``) plus several
+        components of the shape `m \mapsto f(k^t\cdot m +r)` for suitable
+        ``t`` and ``r``.
+
+        Implicitly, the algorithm also maintains a `d \times n_\mathrm{verify}` matrix ``A``
+        (where ``d`` is the dimension of the right vector valued sequence)
+        whose columns are the current right vector valued sequence evaluated at
+        the non-negative integers less than `n_\mathrm{verify}` and ensures that this
+        matrix has full row rank.
+
+        EXAMPLES:
+
+        Binary sum of digits::
+
+            sage: @cached_function
+            ....: def s(n):
+            ....:     if n == 0:
+            ....:         return 0
+            ....:     return s(n//2) + ZZ(is_odd(n))
+            sage: all(s(n) == sum(n.digits(2)) for n in srange(10))
+            True
+            sage: [s(n) for n in srange(10)]
+            [0, 1, 1, 2, 1, 2, 2, 3, 1, 2]
+
+        Let us guess a `2`-linear representation for `s(n)`::
+
+            sage: Seq2 = kRegularSequenceSpace(2, ZZ)
+            sage: import logging
+            sage: logging.basicConfig(level=logging.INFO)
+            sage: S1 = Seq2.guess(s); S1
+            INFO:...:including f_{1*m+0}
+            INFO:...:including f_{2*m+1}
+            2-regular sequence 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, ...
+            sage: S1.linear_representation()
+            ((1, 0),
+             Finite family {0: [1 0]
+                               [0 1],
+                            1: [ 0  1]
+                               [-1  2]},
+             (0, 1))
+
+        The ``INFO`` messages mean that the right vector valued sequence is the sequence `(s(n), s(2n+1))^\top`.
+
+        We guess again, but this time, we use a constant sequence
+        for bootstrapping the guessing process::
+
+            sage: C = Seq2.one_hadamard(); C
+            2-regular sequence 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+            sage: S2 = Seq2.guess(s, sequence=C); S2
+            INFO:...:including 2-regular sequence 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ...
+            INFO:...:including f_{1*m+0}
+            2-regular sequence 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, ...
+            sage: S2.linear_representation()
+            ((0, 1),
+             Finite family {0: [1 0]
+                               [0 1],
+                            1: [1 0]
+                               [1 1]},
+             (1, 0))
+            sage: S1 == S2
+            True
+
+        The sequence of all natural numbers::
+
+            sage: S = Seq2.guess(lambda n: n); S
+            INFO:...:including f_{1*m+0}
+            INFO:...:including f_{2*m+1}
+            2-regular sequence 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ...
+            sage: S.linear_representation()
+            ((1, 0),
+             Finite family {0: [2 0]
+                               [2 1],
+                            1: [ 0  1]
+                               [-2  3]},
+             (0, 1))
+
+        The indicator function of the even integers::
+
+            sage: S = Seq2.guess(lambda n: ZZ(is_even(n))); S
+            INFO:...:including f_{1*m+0}
+            INFO:...:including f_{2*m+0}
+            2-regular sequence 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, ...
+            sage: S.linear_representation()
+            ((1, 0),
+             Finite family {0: [0 1]
+                               [0 1],
+                            1: [0 0]
+                               [0 1]},
+             (1, 1))
+
+        The indicator function of the odd integers::
+
+            sage: S = Seq2.guess(lambda n: ZZ(is_odd(n))); S
+            INFO:...:including f_{1*m+0}
+            INFO:...:including f_{2*m+1}
+            2-regular sequence 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, ...
+            sage: S.linear_representation()
+            ((1, 0),
+             Finite family {0: [0 0]
+                               [0 1],
+                            1: [0 1]
+                               [0 1]},
+             (0, 1))
+
+        TESTS::
+
+            sage: from importlib import reload
+            sage: logging.shutdown(); _ = reload(logging)
+            sage: logging.basicConfig(level=logging.DEBUG)
+            sage: Seq2.guess(s)
+            INFO:...:including f_{1*m+0}
+            DEBUG:...:M_0: f_{2*m+0} = (1) * F_m
+            INFO:...:including f_{2*m+1}
+            DEBUG:...:M_1: f_{2*m+1} = (0, 1) * F_m
+            DEBUG:...:M_0: f_{4*m+1} = (0, 1) * F_m
+            DEBUG:...:M_1: f_{4*m+3} = (-1, 2) * F_m
+            2-regular sequence 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, ...
+            sage: from importlib import reload
+            sage: logging.shutdown(); _ = reload(logging)
+
+        ::
+
+            sage: S = Seq2.guess(lambda n: 2, sequence=C)
+            sage: S
+            2-regular sequence 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, ...
+            sage: S.linear_representation()
+            ((2),
+             Finite family {0: [1],
+                            1: [1]},
+             (1))
+
+        We guess some partial sums sequences::
+
+            sage: S = Seq2((Matrix([1]), Matrix([2])), vector([1]), vector([1]))
+            sage: S
+            2-regular sequence 1, 2, 2, 4, 2, 4, 4, 8, 2, 4, ...
+            sage: from itertools import islice
+            sage: L = []; ps = 0
+            sage: for j in islice(S, 110):
+            ....:     ps += j
+            ....:     L.append(ps)
+            sage: G = Seq2.guess(lambda n: L[n])
+            sage: G
+            2-regular sequence 1, 3, 5, 9, 11, 15, 19, 27, 29, 33, ...
+            sage: G.linear_representation()
+            ((1, 0),
+             Finite family {0: [ 0  1]
+                               [-3  4],
+                            1: [3 0]
+                               [3 2]},
+             (1, 1))
+            sage: G == S.partial_sums(include_n=True)
+            True
+
+        ::
+
+            sage: Seq3 = kRegularSequenceSpace(3, QQ)
+            sage: S = Seq3((Matrix([1]), Matrix([3]), Matrix([2])), vector([1]), vector([1]))
+            sage: S
+            3-regular sequence 1, 3, 2, 3, 9, 6, 2, 6, 4, 3, ...
+            sage: from itertools import islice
+            sage: L = []; ps = 0
+            sage: for j in islice(S, 110):
+            ....:     ps += j
+            ....:     L.append(ps)
+            sage: G = Seq3.guess(lambda n: L[n])
+            sage: G
+            3-regular sequence 1, 4, 6, 9, 18, 24, 26, 32, 36, 39, ...
+            sage: G.linear_representation()
+            ((1, 0),
+             Finite family {0: [ 0  1]
+                               [-6  7],
+                            1: [18/5  2/5]
+                               [18/5 27/5],
+                            2: [ 6  0]
+                               [24  2]},
+             (1, 1))
+            sage: G == S.partial_sums(include_n=True)
+            True
+
+        ::
+
+            sage: Seq2.guess(s, max_exponent=1)
+            Traceback (most recent call last):
+            ...
+            RuntimeError: aborting as exponents would be larger than max_exponent=1
+
+        ::
+
+            sage: R = kRegularSequenceSpace(2, QQ)
+            sage: one = R.one_hadamard()
+            sage: S = R.guess(lambda n: sum(n.bits()), sequence=one) + one
+            sage: T = R.guess(lambda n: n*n, sequence=S, n_verify=4); T
+            2-regular sequence 0, 1, 4, 9, 16, 25, 36, 163/3, 64, 89, ...
+            sage: T.linear_representation()
+            ((0, 0, 1),
+             Finite family {0: [1 0 0]
+                               [0 1 0]
+                               [0 0 4],
+                            1: [   0    1    0]
+                               [  -1    2    0]
+                               [13/3 -5/3 16/3]},
+             (1, 2, 0))
+
+        ::
+
+            sage: two = Seq2.one_hadamard() * 2
+            sage: two.linear_representation()
+            ((1), Finite family {0: [1], 1: [1]}, (2))
+            sage: two_again = Seq2.guess(lambda n: 2, sequence=two)
+            sage: two_again.linear_representation()
+            ((1), Finite family {0: [1], 1: [1]}, (2))
+
+        ::
+
+            sage: def s(k):
+            ....:     return k
+            sage: S1 = Seq2.guess(s)
+            sage: S2 = Seq2.guess(s, sequence=S1)
+            sage: S1
+            2-regular sequence 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ...
+            sage: S2
+            2-regular sequence 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ...
+
+        ::
+
+            sage: A = Seq2(
+            ....:     (Matrix([[1, 1], [1, 1]]), Matrix([[1, 1], [1, 1]])),
+            ....:     left=(1, 1), right=(1, 1))
+            sage: Seq2.guess(lambda n: n, sequence=A, n_verify=5)
+            Traceback (most recent call last):
+            ...
+            RuntimeError: no invertible submatrix found
+        """
+        import logging
+        logger = logging.getLogger(__name__)
+
+        from sage.arith.srange import srange, xsrange
+        from sage.matrix.constructor import Matrix
+        from sage.misc.mrange import cantor_product
+        from sage.modules.free_module_element import vector
+
+        k = self.k
+        domain = self.coefficient_ring()
+        if sequence is None:
+            mu = [[] for _ in srange(k)]
+            seq = lambda m: vector([])
+        else:
+            mu = [M.rows() for M in sequence.mu]
+            seq = lambda m: sequence.coefficient_of_n(m, multiply_left=False)
+            logger.info('including %s', sequence)
+
+        zero = domain(0)
+        one = domain(1)
+
+        # A `line` will be a pair `(t, r)` corresponding to an entry
+        # `k**t * m + r`
+
+        # The elements of `lines` will correspond to the current components
+        # of the right vector valued sequence described in the algorithm section
+        # of the docstring.
+
+        def values(m, lines):
+            """
+                Return current (as defined by ``lines``) right vector valued
+                sequence for argument ``m``.
+            """
+            return tuple(seq(m)) + tuple(f(k**t_R * m + r_R) for t_R, r_R in lines)
+
+        @cached_function(key=lambda lines: len(lines))
+        # we assume that existing lines are not changed
+        # (we allow appending of new lines)
+        def some_inverse_U_matrix(lines):
+            r"""
+                Find an invertible `d \times d` submatrix of the matrix
+                ``A`` described in the algorithm section of the docstring.
+
+                The output is the inverse of the invertible submatrix and
+                the corresponding list of column indices (i.e., arguments to
+                the current right vector valued sequence).
+            """
+            d = len(seq(0)) + len(lines)
+
+            # The following search for an inverse works but is inefficient;
+            # see :trac:`35748` for details.
+            for m_indices in cantor_product(xsrange(n_verify), repeat=d, min_slope=1):
+                # Iterate over all increasing lists of length d consisting
+                # of non-negative integers less than `n_verify`.
+
+                U = Matrix(domain, d, d, [values(m, lines) for m in m_indices]).transpose()
+                try:
+                    return U.inverse(), m_indices
+                except ZeroDivisionError:
+                    pass
+            else:
+                raise RuntimeError('no invertible submatrix found')
+
+        def linear_combination_candidate(t_L, r_L, lines):
+            r"""
+                Based on an invertible submatrix of ``A`` as described in the
+                algorithm section of the docstring, find a candidate for a
+                linear combination of the rows of ``A`` yielding the subsequence
+                with parameters ``t_L`` and ``r_L``, i.e.,
+                `m \mapsto f(k**t_L * m + r_L)`.
+            """
+            iU, m_indices = some_inverse_U_matrix(lines)
+            X_L = vector(f(k**t_L * m + r_L) for m in m_indices)
+            return X_L * iU
+
+        def verify_linear_combination(t_L, r_L, linear_combination, lines):
+            r"""
+                Determine whether the subsequence with parameters ``t_L`` and
+                ``r_L``, i.e., `m \mapsto f(k**t_L * m + r_L)`, is the linear
+                combination ``linear_combination`` of the current vector valued
+                sequence.
+
+                Note that we only evaluate the subsequence of ``f`` where arguments
+                of ``f`` are at most ``n_verify``. This might lead to detection of
+                linear dependence which would not be true for higher values, but this
+                coincides with the documentation of ``n_verify``.
+                However, this is not a guarantee that the given function will never
+                be evaluated beyond ``n_verify``, determining an invertible submatrix
+                in ``some_inverse_U_matrix`` might require us to do so.
+            """
+            return all(f(k**t_L * m + r_L) ==
+                       linear_combination * vector(values(m, lines))
+                       for m in xsrange(0, (n_verify - r_L) // k**t_L + 1))
+
+        class NoLinearCombination(RuntimeError):
+            pass
+
+        def find_linear_combination(t_L, r_L, lines):
+            linear_combination = linear_combination_candidate(t_L, r_L, lines)
+            if not verify_linear_combination(t_L, r_L, linear_combination, lines):
+                raise NoLinearCombination
+            return linear_combination
+
+        if seq(0).is_zero():
+            left = None
+        else:
+            try:
+                left = vector(find_linear_combination(0, 0, []))
+            except NoLinearCombination:
+                left = None
+
+        to_branch = []
+        lines = []
+
+        def include(t, r):
+            to_branch.append((t, r))
+            lines.append((t, r))
+            logger.info('including f_{%s*m+%s}', k**t, r)
+
+        if left is None:
+            include(0, 0)  # entries (t, r) --> k**t * m + r
+            assert len(lines) == 1
+            left = vector(len(seq(0))*(zero,) + (one,))
+
+        while to_branch:
+            t_R, r_R = to_branch.pop(0)
+            if t_R >= max_exponent:
+                raise RuntimeError(f'aborting as exponents would be larger '
+                                   f'than max_exponent={max_exponent}')
+
+            t_L = t_R + 1
+            for s_L in srange(k):
+                r_L = k**t_R * s_L + r_R
+                try:
+                    linear_combination = find_linear_combination(t_L, r_L, lines)
+                except NoLinearCombination:
+                    include(t_L, r_L)  # entries (t, r) --> k**t * m + r
+                    linear_combination = (len(lines)-1)*(zero,) + (one,)
+                logger.debug('M_%s: f_{%s*m+%s} = %s * F_m',
+                             s_L, k**t_L, r_L, linear_combination)
+                mu[s_L].append(linear_combination)
+
+        d = len(seq(0)) + len(lines)
+        mu = tuple(Matrix(domain, [pad_right(tuple(row), d, zero=zero) for row in M])
+                         for M in mu)
+        right = vector(values(0, lines))
+        left = vector(pad_right(tuple(left), d, zero=zero))
+        return self(mu, left, right)
+
     def from_recurrence(self, *args, **kwds):
         r"""
         Construct the unique `k`-regular sequence which fulfills the given


Test patch: