Repository: intellij-rust/intellij-rust (language: rust)

Issue:
Grammar sentence capitalization inspection doesn't work in Rust comments
## Environment

* **IntelliJ Rust plugin version:** 0.4.151.3964-212-nightly
* **Rust toolchain version:** 1.52.0-nightly (b3e19a221 2021-03-12) x86_64-apple-darwin
* **IDE name and version:** CLion 2021.3 EAP (CL-213.777)
* **Operating system:** macOS 10.15.7
* **Macro expansion engine:** new
* **Name resolution engine:** new

## Problem description

While looking into #7131 I noticed the sentence-capitalization inspection reports nothing in Rust comments now. Is it by design?

## Steps to reproduce

1. Enable Preferences -> Editor -> Natural Languages -> Grammar -> Rust -> Comments
2. Enable Preferences -> Editor -> Natural Languages -> Rules -> Capitalization

```rust
fn main() {
    // hello world. how are you harry potter?
    println!("Hello!")
}
```

<img width="688" alt="rust" src="https://user-images.githubusercontent.com/2500320/124382939-97071a00-dcd2-11eb-87b8-6c7f96fdf8cd.png">

Only `harry potter` is highlighted. While in Python file

<img width="709" alt="python" src="https://user-images.githubusercontent.com/2500320/124382954-a8e8bd00-dcd2-11eb-9ca1-be03f92cb9b0.png">



Discussion / hints:

Fix patch:
diff --git a/grazie/src/main/kotlin/org/rust/grazie/RsGrammarCheckingStrategy.kt b/grazie/src/main/kotlin/org/rust/grazie/RsGrammarCheckingStrategy.kt
deleted file mode 100644
index 8313e2b95ee..00000000000
--- a/grazie/src/main/kotlin/org/rust/grazie/RsGrammarCheckingStrategy.kt
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Use of this source code is governed by the MIT license that can be
- * found in the LICENSE file.
- */
-
-// BACKCOMPAT: 2021.1
-@file:Suppress("DEPRECATION")
-
-package org.rust.grazie
-
-import com.intellij.grazie.grammar.strategy.GrammarCheckingStrategy
-import com.intellij.grazie.grammar.strategy.GrammarCheckingStrategy.TextDomain
-import com.intellij.grazie.grammar.strategy.StrategyUtils
-import com.intellij.grazie.grammar.strategy.impl.RuleGroup
-import com.intellij.grazie.utils.LinkedSet
-import com.intellij.lang.injection.InjectedLanguageManager
-import com.intellij.psi.PsiElement
-import org.rust.lang.core.psi.*
-import org.rust.lang.core.psi.ext.elementTypeOrNull
-import org.rust.lang.doc.psi.RsDocComment
-
-class RsGrammarCheckingStrategy : GrammarCheckingStrategy {
-
-    override fun isMyContextRoot(element: PsiElement): Boolean =
-        getContextRootTextDomain(element) != TextDomain.NON_TEXT
-
-    override fun isTypoAccepted(
-        parent: PsiElement,
-        roots: List<PsiElement>,
-        typoRange: IntRange,
-        ruleRange: IntRange
-    ): Boolean {
-        val docCommentRoots = roots.filterIsInstance<RsDocComment>()
-        if (docCommentRoots.isEmpty()) return true
-
-        val injectedLanguageManager = InjectedLanguageManager.getInstance(parent.project)
-        return docCommentRoots.flatMap { it.codeFences }.none {
-            it.textRange.intersects(typoRange.first, typoRange.last) &&
-                !injectedLanguageManager.getInjectedPsiFiles(it).isNullOrEmpty()
-        }
-    }
-
-    override fun getIgnoredRuleGroup(root: PsiElement, child: PsiElement): RuleGroup = RuleGroup.LITERALS
-
-    override fun getStealthyRanges(root: PsiElement, text: CharSequence): LinkedSet<IntRange> {
-        val parent = root.parent
-        return if (parent is RsLitExpr) {
-            val valueTextRange = (parent.kind as? RsLiteralKind.String)?.offsets?.value ?: return linkedSetOf()
-            linkedSetOf(0 until valueTextRange.startOffset, valueTextRange.endOffset until text.length)
-        } else {
-            StrategyUtils.indentIndexes(text, setOf(' ', '/', '!'))
-        }
-    }
-
-    override fun getContextRootTextDomain(root: PsiElement): TextDomain {
-        return when (root.elementTypeOrNull) {
-            in RS_ALL_STRING_LITERALS -> TextDomain.LITERALS
-            in RS_DOC_COMMENTS -> TextDomain.DOCS
-            in RS_REGULAR_COMMENTS -> TextDomain.COMMENTS
-            else -> TextDomain.NON_TEXT
-        }
-    }
-
-    override fun getRootsChain(root: PsiElement): List<PsiElement> {
-        return if (root.elementTypeOrNull in RS_REGULAR_COMMENTS) {
-            StrategyUtils.getNotSoDistantSiblingsOfTypes(this, root, RS_REGULAR_COMMENTS_SET).toList()
-        } else {
-            super.getRootsChain(root)
-        }
-    }
-
-    companion object {
-        private val RS_REGULAR_COMMENTS_SET = RS_REGULAR_COMMENTS.types.toSet()
-    }
-}
diff --git a/grazie/src/main/kotlin/org/rust/grazie/RsTextExtractor.kt b/grazie/src/main/kotlin/org/rust/grazie/RsTextExtractor.kt
new file mode 100644
index 00000000000..080a983dd06
--- /dev/null
+++ b/grazie/src/main/kotlin/org/rust/grazie/RsTextExtractor.kt
@@ -0,0 +1,56 @@
+/*
+ * Use of this source code is governed by the MIT license that can be
+ * found in the LICENSE file.
+ */
+
+package org.rust.grazie
+
+import com.intellij.grazie.text.TextContent
+import com.intellij.grazie.text.TextContent.TextDomain.*
+import com.intellij.grazie.text.TextContentBuilder
+import com.intellij.grazie.text.TextExtractor
+import com.intellij.grazie.utils.getNotSoDistantSimilarSiblings
+import com.intellij.psi.PsiElement
+import com.intellij.psi.impl.source.tree.PsiCommentImpl
+import com.intellij.psi.tree.TokenSet
+import com.intellij.psi.util.elementType
+import org.rust.lang.core.psi.RsLitExpr
+import org.rust.lang.core.psi.RsLiteralKind
+import org.rust.lang.core.psi.kind
+import org.rust.lang.doc.psi.RsDocComment
+import org.rust.lang.doc.psi.RsDocElementTypes
+import org.rust.lang.doc.psi.RsDocElementTypes.DOC_CODE_FENCE
+
+class RsTextExtractor : TextExtractor() {
+
+    override fun buildTextContent(element: PsiElement, allowedDomains: Set<TextContent.TextDomain>): TextContent? {
+        return when {
+            LITERALS in allowedDomains && element is RsLitExpr && element.kind is RsLiteralKind.String -> {
+                TextContentBuilder.FromPsi.build(element, LITERALS)
+            }
+            DOCUMENTATION in allowedDomains && element is RsDocComment-> {
+                DOCUMENTATION_CONTENT_BUILDER.build(element, DOCUMENTATION)
+            }
+            COMMENTS in allowedDomains && element is PsiCommentImpl -> {
+                // Allows extracting single text from a sequence of line comments
+                val siblings = getNotSoDistantSimilarSiblings(element) { it.elementType == element.elementType }
+                TextContent.joinWithWhitespace('\n', siblings.mapNotNull { COMMENT_CONTENT_BUILDER.build(it, COMMENTS) })
+            }
+            else -> null
+        }
+    }
+}
+
+private val DOCUMENTATION_CONTENT_BUILDER: TextContentBuilder = TextContentBuilder.FromPsi
+    .excluding { it.elementType in EXCLUDED_DOCUMENTATION_TOKENS }
+    .withUnknown { it.elementType == DOC_CODE_FENCE }
+    .removingIndents(" \t")
+    .removingLineSuffixes(" \t")
+
+private val COMMENT_CONTENT_BUILDER: TextContentBuilder = TextContentBuilder.FromPsi
+    .removingIndents(" \t/*")
+    .removingLineSuffixes(" \t/*")
+
+private val EXCLUDED_DOCUMENTATION_TOKENS = TokenSet.create(
+    RsDocElementTypes.DOC_GAP
+)
diff --git a/grazie/src/main/resources/org.rust.grazie.xml b/grazie/src/main/resources/org.rust.grazie.xml
index 26de9b5418a..b6e166154a3 100644
--- a/grazie/src/main/resources/org.rust.grazie.xml
+++ b/grazie/src/main/resources/org.rust.grazie.xml
@@ -5,7 +5,8 @@
     </dependencies>
 
     <extensions defaultExtensionNs="com.intellij.grazie">
-        <grammar.strategy language="Rust" implementationClass="org.rust.grazie.RsGrammarCheckingStrategy"/>
+        <textExtractor language="Rust"
+                       implementationClass="org.rust.grazie.RsTextExtractor"/>
         <disableChecking language="Rust"/>
     </extensions>
 </idea-plugin>


Test patch:
diff --git a/grazie/src/test/kotlin/org/rust/grazie/RsGrammarCheckingTest.kt b/grazie/src/test/kotlin/org/rust/grazie/RsGrammarCheckingTest.kt
index 9b5c9a7d640..4d06c3a91f3 100644
--- a/grazie/src/test/kotlin/org/rust/grazie/RsGrammarCheckingTest.kt
+++ b/grazie/src/test/kotlin/org/rust/grazie/RsGrammarCheckingTest.kt
@@ -3,20 +3,13 @@
  * found in the LICENSE file.
  */
 
-// BACKCOMPAT: 2021.1
-@file:Suppress("DEPRECATION")
-
 package org.rust.grazie
 
 import com.intellij.grazie.GrazieConfig
 import com.intellij.grazie.ide.inspection.grammar.GrazieInspection
-import com.intellij.grazie.ide.language.LanguageGrammarChecking
 import com.intellij.grazie.jlanguage.Lang
-import com.intellij.openapi.application.ApplicationInfo
-import com.intellij.openapi.util.BuildNumber
 import com.intellij.openapi.util.Disposer
 import com.intellij.testFramework.PlatformTestUtil
-import com.intellij.util.ThrowableRunnable
 import org.intellij.lang.annotations.Language
 import org.junit.runner.RunWith
 import org.rust.RsJUnit4TestRunner
@@ -30,32 +23,24 @@ class RsGrammarCheckingTest : RsInspectionsTestBase(GrazieInspection::class) {
     override fun createAnnotationFixture(): RsAnnotationTestFixture<Unit> =
         RsAnnotationTestFixture(this, myFixture, inspectionClasses = listOf(inspectionClass), baseFileName = "lib.rs")
 
-    override fun runTestRunnable(testRunnable: ThrowableRunnable<Throwable>) {
-        // TODO: find out why `org.languagetool.JLanguageTool.getBuildDate(JLanguageTool.java:134)` fails
-        //  during `GrazieInitializerManager` initialization
-        if (ApplicationInfo.getInstance().build < BUILD_213) {
-            super.runTestRunnable(testRunnable)
-        }
-    }
-
     override fun setUp() {
         super.setUp()
-        // TODO: find out why `org.languagetool.JLanguageTool.getBuildDate(JLanguageTool.java:134)` fails
-        //  during `GrazieInitializerManager` initialization
-        if (ApplicationInfo.getInstance().build >= BUILD_213) return
-
-        val strategy = LanguageGrammarChecking.getStrategies().first { it is RsGrammarCheckingStrategy }
         val currentState = GrazieConfig.get()
-        if (strategy.getID() !in currentState.enabledGrammarStrategies || currentState.enabledLanguages != enabledLanguages) {
-            updateSettings { state ->
-                val checkingContext = state.checkingContext
-                state.copy(
-                    enabledGrammarStrategies = state.enabledGrammarStrategies + strategy.getID(),
-                    enabledLanguages = enabledLanguages,
-                    checkingContext = checkingContext.copy(enabledLanguages = checkingContext.enabledLanguages + RsLanguage.id)
-                )
-            }
+
+        updateSettings { state ->
+            val checkingContext = state.checkingContext.copy(
+                isCheckInStringLiteralsEnabled = true,
+                isCheckInCommentsEnabled = true,
+                isCheckInDocumentationEnabled = true,
+                enabledLanguages = setOf(RsLanguage.id),
+            )
+            state.copy(
+                enabledLanguages = enabledLanguages,
+                userEnabledRules = setOf("LanguageTool.EN.UPPERCASE_SENTENCE_START"),
+                checkingContext = checkingContext
+            )
         }
+
         Disposer.register(testRootDisposable) {
             updateSettings { currentState }
         }
@@ -63,16 +48,24 @@ class RsGrammarCheckingTest : RsInspectionsTestBase(GrazieInspection::class) {
 
     fun `test check literals`() = doTest("""
         fn foo() {
-            let literal = "It is <TYPO>friend</TYPO> of human";
-            let raw_literal = r"It is <TYPO>friend</TYPO> of human";
-            let binary_literal = b"It is <TYPO>friend</TYPO> of human";
+            let literal = "It is <GRAMMAR_ERROR>an</GRAMMAR_ERROR> friend of human";
+            let raw_literal = r"It is <GRAMMAR_ERROR>an</GRAMMAR_ERROR> friend of human";
+            let binary_literal = b"It is <GRAMMAR_ERROR>an</GRAMMAR_ERROR> friend of human";
         }
     """, checkInStringLiterals = true)
 
+    // https://github.com/intellij-rust/intellij-rust/issues/7446
+    fun `test sentence capitalization in comments`() = doTest("""
+        fn main() {
+            // <GRAMMAR_ERROR>hello</GRAMMAR_ERROR> world. <GRAMMAR_ERROR>how</GRAMMAR_ERROR> are you Harry Potter?
+            println!("Hello!")
+        }
+    """, checkInComments = true)
+
     fun `test check comments`() = doTest("""
         fn foo() {
-            // It is <TYPO>friend</TYPO> of human
-            /* It is <TYPO>friend</TYPO> of human */
+            // It is <GRAMMAR_ERROR>an</GRAMMAR_ERROR> friend of human
+            /* It is <GRAMMAR_ERROR>an</GRAMMAR_ERROR> friend of human */
             let literal = 123;
         }
     """, checkInComments = true)
@@ -80,11 +73,11 @@ class RsGrammarCheckingTest : RsInspectionsTestBase(GrazieInspection::class) {
     // https://github.com/intellij-rust/intellij-rust/issues/7024
     fun `test check single sentence in sequential comments 1`() = doTest("""
         fn main() {
-            // Path to directory where someone <TYPO>write</TYPO>
-            // and from where someone reads
+            // With <GRAMMAR_ERROR>you</GRAMMAR_ERROR>
+            // I can finally be happy
             let path1 = "/foo/bar";
-            /* Path to directory where someone <TYPO>write</TYPO> */
-            /* and from where someone reads */
+            /* With <GRAMMAR_ERROR>you</GRAMMAR_ERROR> */
+            /* I can finally be happy */
             let path2 = "/foo/bar";
         }
     """, checkInComments = true)
@@ -92,19 +85,19 @@ class RsGrammarCheckingTest : RsInspectionsTestBase(GrazieInspection::class) {
     // https://github.com/intellij-rust/intellij-rust/issues/7024
     fun `test check single sentence in sequential comments 2`() = doTest("""
         fn main() {
-            // Path to directory where someone writes
+            // With you
 
-            // <TYPE>and</TYPE> from where someone reads
+            // I can finally be happy
             let path = "/foo/bar";
         }
     """, checkInComments = true)
 
     fun `test check doc comments`() = doTest("""
-        /// It is <TYPO>friend</TYPO> of human
+        /// It is <GRAMMAR_ERROR>an</GRAMMAR_ERROR> friend of human
         mod foo {
-            //! It is <TYPO>friend</TYPO> of human
+            //! It is <GRAMMAR_ERROR>an</GRAMMAR_ERROR> friend of human
 
-            /** It is <TYPO>friend</TYPO> of human */
+            /** It is <GRAMMAR_ERROR>an</GRAMMAR_ERROR> friend of human */
             fn bar() {}
         }
     """, checkInDocumentation = true)
@@ -112,7 +105,7 @@ class RsGrammarCheckingTest : RsInspectionsTestBase(GrazieInspection::class) {
     fun `test check injected Rust code in doc comments`() = doTest("""
         ///
         /// ```
-        /// let literal = "It is <TYPO>friend</TYPO> of human";
+        /// let literal = "It is <GRAMMAR_ERROR>an</GRAMMAR_ERROR> friend of human";
         /// for i in 1..10 {}
         /// ```
         pub fn foo() {}
@@ -151,7 +144,7 @@ class RsGrammarCheckingTest : RsInspectionsTestBase(GrazieInspection::class) {
             state.copy(checkingContext = newContext)
         }
 
-        checkByText(text.replace("<TYPO.*?>(.*?)</TYPO>".toRegex(), "$1"))
+        checkByText(text.replace("<GRAMMAR_ERROR.*?>(.*?)</GRAMMAR_ERROR>".toRegex(), "$1"))
     }
 
     private fun updateSettings(change: (GrazieConfig.State) -> GrazieConfig.State) {
@@ -161,6 +154,5 @@ class RsGrammarCheckingTest : RsInspectionsTestBase(GrazieInspection::class) {
 
     companion object {
         private val enabledLanguages = setOf(Lang.AMERICAN_ENGLISH)
-        private val BUILD_213: BuildNumber = BuildNumber.fromString("213")!!
     }
 }