Repository: matthewwardrop/formulaic

Issue:
LinearConstraint parser still fails on Categorical variables
On main I'm still seeing failures in the code below.  Having this work correctly on terms like `C(var)[T.value]` is practically an important case since `C()` is common when `[T.value]` is present. The tests only look at cases like `A[T.value]`.

```python
import formulaic

constraints = "C(agecat)[T.4] = C(agecat)[T.5]"
variable_names = [
    "Intercept",
    "C(agecat)[T.2]",
    "C(agecat)[T.3]",
    "C(agecat)[T.4]",
    "C(agecat)[T.5]",
    "logpyears",
    "smokes",
]

lc = formulaic.utils.constraints.LinearConstraints.from_spec(
    constraints, variable_names=variable_names
)

```


Exception is 
```
---------------------------------------------------------------------------
FormulaSyntaxError                        Traceback (most recent call last)
Cell In[2], line 14
      3 constraints = "C(agecat)[T.4] = C(agecat)[T.5]"
      4 variable_names = [
      5     "Intercept",
      6     "C(agecat)[T.2]",
   (...)
     11     "smokes",
     12 ]
---> 14 lc = formulaic.utils.constraints.LinearConstraints.from_spec(
     15     constraints, variable_names=variable_names
     16 )

File C:\git\formulaic\formulaic\utils\constraints.py:102, in LinearConstraints.from_spec(cls, spec, variable_names)
     98     spec = ",".join(spec)
     99 if isinstance(spec, str):
    100     matrix, values = LinearConstraintParser(
    101         variable_names=variable_names
--> 102     ).get_matrix(spec)
    103     return cls(matrix, values, variable_names)
    104 matrices, constants = [], []

File C:\git\formulaic\formulaic\utils\constraints.py:277, in LinearConstraintParser.get_matrix(self, formula)
    263 def get_matrix(
    264     self, formula: str
    265 ) -> Tuple["numpy.typing.ArrayLike", "numpy.typing.ArrayLike"]:
    266     """
    267     Build the constraint matrix and constraint values vector associated with
    268     the parsed string.
   (...)
    275         A tuple of the contraint matrix and constraint values respectively.
    276     """
--> 277     constraints = self.get_terms(formula)
    278     if not constraints:
    279         return numpy.empty((0, len(self.variable_names))), numpy.array([])

File C:\git\formulaic\formulaic\utils\constraints.py:255, in LinearConstraintParser.get_terms(self, formula)
    246 def get_terms(
    247     self, formula: str
    248 ) -> Union[None, List[ScaledFactor], Tuple[List[ScaledFactor], ...]]:
    249     """
    250     Build the `ScaledFactor` instances for a constraint formula string.
    251
    252     Args:
    253         formula: The constraint formula for which to build terms.
    254     """
--> 255     ast = self.get_ast(formula)
    256     if not ast:
    257         return None

File C:\git\formulaic\formulaic\utils\constraints.py:240, in LinearConstraintParser.get_ast(self, formula)
    230 def get_ast(self, formula: str) -> Optional[ASTNode]:
    231     """
    232     Assemble an abstract syntax tree for the nominated `formula` string.
    233
   (...)
    236             generated.
    237     """
    238     return cast(
    239         Optional[ASTNode],
--> 240         tokens_to_ast(
    241             self.get_tokens(formula),
    242             operator_resolver=self.operator_resolver,
    243         ),
    244     )

File C:\git\formulaic\formulaic\parser\algos\tokens_to_ast.py:137, in tokens_to_ast(tokens, operator_resolver)
    135 if output_queue:
    136     if len(output_queue) > 1:
--> 137         raise exc_for_missing_operator(output_queue[0], output_queue[1])
    138     return output_queue[0]
    140 return None

FormulaSyntaxError: Missing operator between `C(agecat)` and `T.4`.

⧛C(agecat)[T.4⧚] = C(agecat)[T.5]
```

Discussion:
@bashtage: Sorry to report that the previous patch didn't quite get to the full set of issues. @matthewwardrop: Ah! Thanks @bashtage ! Having you hammering it in real-world use-cases has been really helpful in getting the bug- and edge-counts down... no need to apologise!

Will fix this soon!

Solution patch:
diff --git a/formulaic/parser/algos/tokenize.py b/formulaic/parser/algos/tokenize.py
index 8c73bfe1..1d08526d 100644
--- a/formulaic/parser/algos/tokenize.py
+++ b/formulaic/parser/algos/tokenize.py
@@ -79,14 +79,14 @@ def tokenize(
         if quote_context and char == quote_context[-1]:
             token.update(char, i)
             quote_context.pop(-1)
-            if (
-                token
-                and not quote_context
-                and token.kind is Token.Kind.PYTHON
-                and char in ("]", ")")
-            ):
-                yield token
-                token = Token(source=formula)
+            # if (
+            #     token
+            #     and not quote_context
+            #     and token.kind is Token.Kind.PYTHON
+            #     and char in ("]", ")")
+            # ):
+            #     yield token
+            #     token = Token(source=formula)
             continue
         if quote_context and quote_context[-1] in ('"', "'", "`", ")", "]", "}", "%"):
             if char in "`([" and quote_context[-1] in "})]":
@@ -150,9 +150,11 @@ def tokenize(
             continue  # pragma: no cover; workaround bug in coverage
 
         if word_chars.match(char):
+            if token and token.kind in (Token.Kind.OPERATOR, Token.Kind.PYTHON):
+                yield token
+                token = Token(source=formula)
             if token.kind not in (
                 None,
-                Token.Kind.OPERATOR,
                 Token.Kind.VALUE,
                 Token.Kind.NAME,
             ):
@@ -160,9 +162,6 @@ def tokenize(
                     Token(source=formula, source_start=i, source_end=i),
                     f"Unexpected token kind {token.kind} for character '{char}'.",
                 )
-            if token and token.kind is Token.Kind.OPERATOR:
-                yield token
-                token = Token(source=formula)
             if numeric_chars.match(char) and token.kind in (None, Token.Kind.VALUE):
                 kind = "value"
             else:


Test patch:
diff --git a/tests/parser/algos/test_tokenize.py b/tests/parser/algos/test_tokenize.py
index 512884ae..f0ca2e4f 100644
--- a/tests/parser/algos/test_tokenize.py
+++ b/tests/parser/algos/test_tokenize.py
@@ -67,6 +67,10 @@
     "A(T.a)": [
         "python:A(T.a)",
     ],
+    "A(T.a)[b](c)": [
+        "python:A(T.a)[b](c)",
+    ],
+    "A(T.a)[b](c)a": ["python:A(T.a)[b](c)", "name:a"],
 }
 
 TOKEN_ERRORS = {