File: Mathlib/Tactic/RewriteSearch.lean

full_name: Mathlib.Tactic.RewriteSearch.initFn._@.Mathlib.Tactic.RewriteSearch._hyg.8
code: initialize registerTraceClass `rw_search
goal: IO Unit

full_name: Mathlib.Tactic.RewriteSearch.initFn._@.Mathlib.Tactic.RewriteSearch._hyg.49
code: initialize registerTraceClass `rw_search.detail
goal: IO Unit

full_name: Mathlib.Tactic.RewriteSearch.splitDelimiters.auxEnd
code: /-- Separate a string into a list of strings by pulling off initial `(` or `]` characters,
and pulling off terminal `)`, `]`, or `,` characters. -/
partial def splitDelimiters (s : String) : List String 
goal: String.Pos × List String

full_name: Mathlib.Tactic.RewriteSearch.splitDelimiters
code: /-- Separate a string into a list of strings by pulling off initial `(` or `]` characters,
and pulling off terminal `)`, `]`, or `,` characters. -/
partial def splitDelimiters (s : String) : List String 
goal: List String

full_name: Mathlib.Tactic.RewriteSearch.splitDelimiters.auxStart
code: /-- Separate a string into a list of strings by pulling off initial `(` or `]` characters,
and pulling off terminal `)`, `]`, or `,` characters. -/
partial def splitDelimiters (s : String) : List String 
goal: String.Pos × List String

full_name: Mathlib.Tactic.RewriteSearch.tokenize
code: /--
Tokenize a string at whitespace, and then pull off delimiters.
-/
-- `rw_search` seems to return better results if we tokenize
-- rather than just breaking the string into characters,
-- but more extensive testing would be great!
-- Tokenizing of course makes the edit distance calculations cheaper,
-- because the lists are significantly shorter.
-- On the other hand, because we currently use unweighted edit distance,
-- this makes it "cheap" to change one identifier for another.
def tokenize (e : Expr) : MetaM (List String) 
goal: Lean.MetaM (List String)

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.history
code: /--
Data structure containing the history of a rewrite search.
-/
structure SearchNode where mk' ::
  /-- The lemmas used so far. -/
  -- The first component is the index amongst successful rewrites at the previous node.
  history : Array (Nat × Expr × Bool)
  /-- The metavariable context after rewriting.
  We carry this around so the search can safely backtrack. -/
  mctx : MetavarContext
  /-- The current goal. -/
  goal : MVarId
  /-- The type of the current goal. -/
  type : Expr
  /-- The pretty printed current goal. -/
  ppGoal : String
  /-- The tokenization of the left-hand-side of the current goal. -/
  lhs : List String
  /-- The tokenization of the right-hand-side of the current goal. -/
  rhs : List String
  /-- Whether the current goal can be closed by `rfl` (or `none` if this hasn't been test yet). -/
  rfl? : Option Bool := none
  /-- The edit distance between the tokenizations of the two sides
  (or `none` if this hasn't been computed yet). -/
  dist? : Option Nat := none
goal: Array (ℕ × Lean.Expr × Bool)

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.dist?
code: /--
Data structure containing the history of a rewrite search.
-/
structure SearchNode where mk' ::
  /-- The lemmas used so far. -/
  -- The first component is the index amongst successful rewrites at the previous node.
  history : Array (Nat × Expr × Bool)
  /-- The metavariable context after rewriting.
  We carry this around so the search can safely backtrack. -/
  mctx : MetavarContext
  /-- The current goal. -/
  goal : MVarId
  /-- The type of the current goal. -/
  type : Expr
  /-- The pretty printed current goal. -/
  ppGoal : String
  /-- The tokenization of the left-hand-side of the current goal. -/
  lhs : List String
  /-- The tokenization of the right-hand-side of the current goal. -/
  rhs : List String
  /-- Whether the current goal can be closed by `rfl` (or `none` if this hasn't been test yet). -/
  rfl? : Option Bool := none
  /-- The edit distance between the tokenizations of the two sides
  (or `none` if this hasn't been computed yet). -/
  dist? : Option Nat := none
goal: Option ℕ

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.mk'
code: /--
Data structure containing the history of a rewrite search.
-/
structure SearchNode where mk' ::
  /-- The lemmas used so far. -/
  -- The first component is the index amongst successful rewrites at the previous node.
  history : Array (Nat × Expr × Bool)
  /-- The metavariable context after rewriting.
  We carry this around so the search can safely backtrack. -/
  mctx : MetavarContext
  /-- The current goal. -/
  goal : MVarId
  /-- The type of the current goal. -/
  type : Expr
  /-- The pretty printed current goal. -/
  ppGoal : String
  /-- The tokenization of the left-hand-side of the current goal. -/
  lhs : List String
  /-- The tokenization of the right-hand-side of the current goal. -/
  rhs : List String
  /-- Whether the current goal can be closed by `rfl` (or `none` if this hasn't been test yet). -/
  rfl? : Option Bool := none
  /-- The edit distance between the tokenizations of the two sides
  (or `none` if this hasn't been computed yet). -/
  dist? : Option Nat := none
goal: Mathlib.Tactic.RewriteSearch.SearchNode

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.rhs
code: /--
Data structure containing the history of a rewrite search.
-/
structure SearchNode where mk' ::
  /-- The lemmas used so far. -/
  -- The first component is the index amongst successful rewrites at the previous node.
  history : Array (Nat × Expr × Bool)
  /-- The metavariable context after rewriting.
  We carry this around so the search can safely backtrack. -/
  mctx : MetavarContext
  /-- The current goal. -/
  goal : MVarId
  /-- The type of the current goal. -/
  type : Expr
  /-- The pretty printed current goal. -/
  ppGoal : String
  /-- The tokenization of the left-hand-side of the current goal. -/
  lhs : List String
  /-- The tokenization of the right-hand-side of the current goal. -/
  rhs : List String
  /-- Whether the current goal can be closed by `rfl` (or `none` if this hasn't been test yet). -/
  rfl? : Option Bool := none
  /-- The edit distance between the tokenizations of the two sides
  (or `none` if this hasn't been computed yet). -/
  dist? : Option Nat := none
goal: List String

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.type
code: /--
Data structure containing the history of a rewrite search.
-/
structure SearchNode where mk' ::
  /-- The lemmas used so far. -/
  -- The first component is the index amongst successful rewrites at the previous node.
  history : Array (Nat × Expr × Bool)
  /-- The metavariable context after rewriting.
  We carry this around so the search can safely backtrack. -/
  mctx : MetavarContext
  /-- The current goal. -/
  goal : MVarId
  /-- The type of the current goal. -/
  type : Expr
  /-- The pretty printed current goal. -/
  ppGoal : String
  /-- The tokenization of the left-hand-side of the current goal. -/
  lhs : List String
  /-- The tokenization of the right-hand-side of the current goal. -/
  rhs : List String
  /-- Whether the current goal can be closed by `rfl` (or `none` if this hasn't been test yet). -/
  rfl? : Option Bool := none
  /-- The edit distance between the tokenizations of the two sides
  (or `none` if this hasn't been computed yet). -/
  dist? : Option Nat := none
goal: Lean.Expr

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.goal
code: /--
Data structure containing the history of a rewrite search.
-/
structure SearchNode where mk' ::
  /-- The lemmas used so far. -/
  -- The first component is the index amongst successful rewrites at the previous node.
  history : Array (Nat × Expr × Bool)
  /-- The metavariable context after rewriting.
  We carry this around so the search can safely backtrack. -/
  mctx : MetavarContext
  /-- The current goal. -/
  goal : MVarId
  /-- The type of the current goal. -/
  type : Expr
  /-- The pretty printed current goal. -/
  ppGoal : String
  /-- The tokenization of the left-hand-side of the current goal. -/
  lhs : List String
  /-- The tokenization of the right-hand-side of the current goal. -/
  rhs : List String
  /-- Whether the current goal can be closed by `rfl` (or `none` if this hasn't been test yet). -/
  rfl? : Option Bool := none
  /-- The edit distance between the tokenizations of the two sides
  (or `none` if this hasn't been computed yet). -/
  dist? : Option Nat := none
goal: Lean.MVarId

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.ppGoal
code: /--
Data structure containing the history of a rewrite search.
-/
structure SearchNode where mk' ::
  /-- The lemmas used so far. -/
  -- The first component is the index amongst successful rewrites at the previous node.
  history : Array (Nat × Expr × Bool)
  /-- The metavariable context after rewriting.
  We carry this around so the search can safely backtrack. -/
  mctx : MetavarContext
  /-- The current goal. -/
  goal : MVarId
  /-- The type of the current goal. -/
  type : Expr
  /-- The pretty printed current goal. -/
  ppGoal : String
  /-- The tokenization of the left-hand-side of the current goal. -/
  lhs : List String
  /-- The tokenization of the right-hand-side of the current goal. -/
  rhs : List String
  /-- Whether the current goal can be closed by `rfl` (or `none` if this hasn't been test yet). -/
  rfl? : Option Bool := none
  /-- The edit distance between the tokenizations of the two sides
  (or `none` if this hasn't been computed yet). -/
  dist? : Option Nat := none
goal: String

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.rfl?
code: /--
Data structure containing the history of a rewrite search.
-/
structure SearchNode where mk' ::
  /-- The lemmas used so far. -/
  -- The first component is the index amongst successful rewrites at the previous node.
  history : Array (Nat × Expr × Bool)
  /-- The metavariable context after rewriting.
  We carry this around so the search can safely backtrack. -/
  mctx : MetavarContext
  /-- The current goal. -/
  goal : MVarId
  /-- The type of the current goal. -/
  type : Expr
  /-- The pretty printed current goal. -/
  ppGoal : String
  /-- The tokenization of the left-hand-side of the current goal. -/
  lhs : List String
  /-- The tokenization of the right-hand-side of the current goal. -/
  rhs : List String
  /-- Whether the current goal can be closed by `rfl` (or `none` if this hasn't been test yet). -/
  rfl? : Option Bool := none
  /-- The edit distance between the tokenizations of the two sides
  (or `none` if this hasn't been computed yet). -/
  dist? : Option Nat := none
goal: Option Bool

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.mctx
code: /--
Data structure containing the history of a rewrite search.
-/
structure SearchNode where mk' ::
  /-- The lemmas used so far. -/
  -- The first component is the index amongst successful rewrites at the previous node.
  history : Array (Nat × Expr × Bool)
  /-- The metavariable context after rewriting.
  We carry this around so the search can safely backtrack. -/
  mctx : MetavarContext
  /-- The current goal. -/
  goal : MVarId
  /-- The type of the current goal. -/
  type : Expr
  /-- The pretty printed current goal. -/
  ppGoal : String
  /-- The tokenization of the left-hand-side of the current goal. -/
  lhs : List String
  /-- The tokenization of the right-hand-side of the current goal. -/
  rhs : List String
  /-- Whether the current goal can be closed by `rfl` (or `none` if this hasn't been test yet). -/
  rfl? : Option Bool := none
  /-- The edit distance between the tokenizations of the two sides
  (or `none` if this hasn't been computed yet). -/
  dist? : Option Nat := none
goal: Lean.MetavarContext

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.lhs
code: /--
Data structure containing the history of a rewrite search.
-/
structure SearchNode where mk' ::
  /-- The lemmas used so far. -/
  -- The first component is the index amongst successful rewrites at the previous node.
  history : Array (Nat × Expr × Bool)
  /-- The metavariable context after rewriting.
  We carry this around so the search can safely backtrack. -/
  mctx : MetavarContext
  /-- The current goal. -/
  goal : MVarId
  /-- The type of the current goal. -/
  type : Expr
  /-- The pretty printed current goal. -/
  ppGoal : String
  /-- The tokenization of the left-hand-side of the current goal. -/
  lhs : List String
  /-- The tokenization of the right-hand-side of the current goal. -/
  rhs : List String
  /-- Whether the current goal can be closed by `rfl` (or `none` if this hasn't been test yet). -/
  rfl? : Option Bool := none
  /-- The edit distance between the tokenizations of the two sides
  (or `none` if this hasn't been computed yet). -/
  dist? : Option Nat := none
goal: List String

full_name: Mathlib.Tactic.RewriteSearch.SearchNode
code: /--
Data structure containing the history of a rewrite search.
-/
structure SearchNode where mk' ::
  /-- The lemmas used so far. -/
  -- The first component is the index amongst successful rewrites at the previous node.
  history : Array (Nat × Expr × Bool)
  /-- The metavariable context after rewriting.
  We carry this around so the search can safely backtrack. -/
  mctx : MetavarContext
  /-- The current goal. -/
  goal : MVarId
  /-- The type of the current goal. -/
  type : Expr
  /-- The pretty printed current goal. -/
  ppGoal : String
  /-- The tokenization of the left-hand-side of the current goal. -/
  lhs : List String
  /-- The tokenization of the right-hand-side of the current goal. -/
  rhs : List String
  /-- Whether the current goal can be closed by `rfl` (or `none` if this hasn't been test yet). -/
  rfl? : Option Bool := none
  /-- The edit distance between the tokenizations of the two sides
  (or `none` if this hasn't been computed yet). -/
  dist? : Option Nat := none
goal: Type

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.editCost
code: /--
What is the cost for changing a token?
`Levenshtein.defaultCost` just uses constant cost `1` for any token.

It may be interesting to try others.
the only one I've experimented with so far is `Levenshtein.stringLogLengthCost`,
which performs quite poorly!
-/
def editCost : Levenshtein.Cost String String Nat 
goal: Levenshtein.Cost String String ℕ

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.compute_rfl?
code: /-- Check whether a goal can be solved by `rfl`, and fill in the `SearchNode.rfl?` field. -/
def compute_rfl? (n : SearchNode) : MetaM SearchNode 
goal: Lean.MetaM Mathlib.Tactic.RewriteSearch.SearchNode

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.compute_dist?
code: /-- Fill in the `SearchNode.dist?` field with the edit distance between the two sides. -/
def compute_dist? (n : SearchNode) : SearchNode 
goal: Mathlib.Tactic.RewriteSearch.SearchNode

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.toString
code: /-- Represent a search node as string, solely for debugging. -/
def toString (n : SearchNode) : MetaM String 
goal: Lean.MetaM String

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.mk
code: /-- Construct a `SearchNode`. -/
def mk (history : Array (Nat × Expr × Bool)) (goal : MVarId) (ctx : Option MetavarContext := none) :
    MetaM (Option SearchNode) 
goal: Lean.MetaM (Option Mathlib.Tactic.RewriteSearch.SearchNode)

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.init
code: /-- Construct an initial `SearchNode` from a goal. -/
def init (goal : MVarId) : MetaM (Option SearchNode) 
goal: Lean.MetaM (Option Mathlib.Tactic.RewriteSearch.SearchNode)

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.push
code: /-- Add an additional step to the `SearchNode` history. -/
def push (n : SearchNode) (expr : Expr) (symm : Bool) (k : Nat) (g : MVarId)
    (ctx : Option MetavarContext := none) : MetaM (Option SearchNode) 
goal: Lean.MetaM (Option Mathlib.Tactic.RewriteSearch.SearchNode)

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.lastIdx
code: /-- Report the index of the most recently applied lemma, in the ordering returned by `rw?`. -/
def lastIdx (n : SearchNode) : Nat 
goal: ℕ

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.instOrd
code: instance : Ord SearchNode 
goal: Ord Mathlib.Tactic.RewriteSearch.SearchNode

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.penalty
code: /--
A somewhat arbitrary penalty function.
Note that `n.lastIdx` penalizes using later lemmas from a particular call to `rw?` at a node,
but once we have moved on to the next node these penalties are "forgiven".

(You might in interpret this as encouraging
the algorithm to "trust" the ordering provided by `rw?`.)

I tried out a various (positive) linear combinations of
`.history.size`, `.lastIdx`, and `.ppGoal.length` (and also the `.log2`s of these).
* `.lastIdx.log2` is quite good, and the best coefficient is around 1.
* `.lastIdx / 10` is almost as good.
* `.history.size` makes things worse (similarly with `.log2`).
* `.ppGoal.length` makes little difference (similarly with `.log2`).
Here testing consisting of running the current `rw_search` test suite,
rejecting values for which any failed, and trying to minimize the run time reported by
```shell
lake build &&  \
time (lake env lean test/RewriteSearch/Basic.lean; \
  lake env lean test/RewriteSearch/Polynomial.lean)
```

With a larger test suite it might be worth running this minimization again,
and considering other penalty functions.

(If you do this, please choose a penalty function which is in the interior of the region
where the test suite works.
I think it would be a bad idea to optimize the run time at the expense of fragility.)
-/
def penalty (n : SearchNode) : Nat 
goal: ℕ

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.prio
code: /-- The priority function for search is Levenshtein distance plus a penalty. -/
abbrev prio (n : SearchNode) : Thunk Nat 
goal: Thunk ℕ

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.estimator
code: /-- We can obtain lower bounds, and improve them, for the Levenshtein distance. -/
abbrev estimator (n : SearchNode) : Type 
goal: Type

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.rewrite
code: /-- Given a `RewriteResult` from the `rw?` tactic, create a new `SearchNode` with the new goal. -/
def rewrite (n : SearchNode) (r : Rewrites.RewriteResult) (k : Nat) :
    MetaM (Option SearchNode) 
goal: Lean.MetaM (Option Mathlib.Tactic.RewriteSearch.SearchNode)

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.rewrites
code: /--
Given a pair of `DiscrTree` trees
indexing all rewrite lemmas in the imported files and the current file,
try rewriting the current goal in the `SearchNode` by one of them,
returning a `MLList MetaM SearchNode`, i.e. a lazy list of next possible goals.
-/
def rewrites (hyps : Array (Expr × Bool × Nat))
    (lemmas : ModuleDiscrTreeRef (Name × RwDirection))
    (forbidden : NameSet := ∅) (n : SearchNode) : MLList MetaM SearchNode 
goal: MLList Lean.MetaM Mathlib.Tactic.RewriteSearch.SearchNode

full_name: Mathlib.Tactic.RewriteSearch.SearchNode.search
code: /--
Perform best first search on the graph of rewrites from the specified `SearchNode`.
-/
def search (n : SearchNode)
    (stopAtRfl := true) (stopAtDistZero := true)
    (forbidden : NameSet := ∅) (maxQueued : Option Nat := none) :
    MLList MetaM SearchNode 
goal: MLList Lean.MetaM Mathlib.Tactic.RewriteSearch.SearchNode

full_name: Mathlib.Tactic.RewriteSearch.tacticRw_search_
code: /--
`rw_search` attempts to solve an equality goal
by repeatedly rewriting using lemmas from the library.

If no solution is found,
the best sequence of rewrites found before `maxHeartbeats` elapses is returned.

The search is a best-first search, minimising the Levenshtein edit distance between
the pretty-printed expressions on either side of the equality.
(The strings are tokenized at spaces,
separating delimiters `(`, `)`, `[`, `]`, and `,` into their own tokens.)

You can use `rw_search [-my_lemma, -my_theorem]`
to prevent `rw_search` from using the names theorems.
-/
syntax "rw_search" (rewrites_forbidden)? : tactic
goal: Lean.ParserDescr

full_name: Mathlib.Tactic.RewriteSearch._aux_Mathlib_Tactic_RewriteSearch___elabRules_Mathlib_Tactic_RewriteSearch_tacticRw_search__1
code: elab_rules : tactic |
    `(tactic| rw_search%$tk $[[ $[-$forbidden],* ]]?) => withMainContext do
  let forbidden : NameSet :=
    ((forbidden.getD #[]).map Syntax.getId).foldl (init := ∅) fun s n => s.insert n
  let .some init ← SearchNode.init (← getMainGoal) | throwError "Goal is not an equality."
  let results := init.search (forbidden := forbidden) |>.whileAtLeastHeartbeatsPercent 20
  let results ← results.force
  let min ← match results with
  | [] => failure
  | h :: t =>
      pure <| t.foldl (init := h) fun r₁ r₂ =>
        if r₁.rfl? = some true then r₁
        else if r₂.rfl? = some true then r₂
        else if r₁.dist?.getD 0 ≤ r₂.dist?.getD 0 then r₁ else r₂
  setMCtx min.mctx
  replaceMainGoal [min.goal]
  let type? ← if min.rfl? = some true then pure none else do pure <| some (← min.goal.getType)
  addRewriteSuggestion tk (min.history.toList.map (·.2))
    type? (origSpan? := ← getRef)
goal: Lean.Elab.Tactic.Tactic