Repository: checkstyle/checkstyle

Problem statement:
Metadata generation fails on line wrapped type description
Discovered at https://github.com/checkstyle/checkstyle/pull/9076#discussion_r559056728, when a check's class level javadoc  property description is line wrapped in  mid-sentence, metadata generation is incomplete.

From https://github.com/checkstyle/checkstyle/pull/9076:

```java

 * Property {@code requiredJavadocPhrase} - Specify the comment text pattern which qualifies a
 * method as designed for extension. Supports multi-line regex. Type is
 * {@code java.util.regex.Pattern}.
 * Default value is {@code ".*"}.

```

Yields:
```xml

            <property default-value=".*" name="requiredJavadocPhrase" type="">
               <description/>
            </property>

```
Also,

```java

 * Property {@code requiredJavadocPhrase} - Specify the comment text pattern which qualifies a
 * method as designed for extension. Type is {@code java.util.regex.Pattern}.
 * Default value is {@code ".*"}.
 * 
```

Yields:
```xml

            <property default-value=".*"
                      name="requiredJavadocPhrase"
                      type="java.util.regex.Pattern">
               <description>Specify the comment text pattern which qualifies a</description>
            </property>

```
-----

**The following is outdated and incorrect, and can be fixed at https://github.com/checkstyle/checkstyle/pull/9209:**

During a fix remove from configs:

In `config/checkstyle_non_main_files_suppressions.xml`:
```xml
<!-- until https://github.com/checkstyle/checkstyle/issues/9204 -->
   <suppress id="lineLength" files="config_design\.xml" lines="163"/>
``` 

In `config/suppressions.xml`:
```xml
  <!-- until https://github.com/checkstyle/checkstyle/issues/9204 -->
  <suppress id="lineLengthXml" files="config_design\.xml" lines="163"/>
```


Hints:

Patch:
diff --git a/src/main/java/com/puppycrawl/tools/checkstyle/checks/javadoc/JavadocMissingLeadingAsteriskCheck.java b/src/main/java/com/puppycrawl/tools/checkstyle/checks/javadoc/JavadocMissingLeadingAsteriskCheck.java
--- a/src/main/java/com/puppycrawl/tools/checkstyle/checks/javadoc/JavadocMissingLeadingAsteriskCheck.java
+++ b/src/main/java/com/puppycrawl/tools/checkstyle/checks/javadoc/JavadocMissingLeadingAsteriskCheck.java
@@ -41,7 +41,8 @@
  * Property {@code violateExecutionOnNonTightHtml} - Control when to print violations if the
  * Javadoc being examined by this check violates the tight html rules defined at
  * <a href="https://checkstyle.org/writingjavadocchecks.html#Tight-HTML_rules">Tight-HTML Rules</a>.
- * Type is {@code boolean}. Default value is {@code false}.
+ * Type is {@code boolean}.
+ * Default value is {@code false}.
  * </li>
  * </ul>
  * <p>
diff --git a/src/main/java/com/puppycrawl/tools/checkstyle/meta/JavadocMetadataScraper.java b/src/main/java/com/puppycrawl/tools/checkstyle/meta/JavadocMetadataScraper.java
--- a/src/main/java/com/puppycrawl/tools/checkstyle/meta/JavadocMetadataScraper.java
+++ b/src/main/java/com/puppycrawl/tools/checkstyle/meta/JavadocMetadataScraper.java
@@ -26,6 +26,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashSet;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -57,14 +58,14 @@ public class JavadocMetadataScraper extends AbstractJavadocCheck {
     private static final Pattern PROPERTY_TAG = Pattern.compile("\\s*Property\\s*");
 
     /** Regular expression for property type location in class-level javadocs. */
-    private static final Pattern TYPE_TAG = Pattern.compile("\\s.*Type is\\s.*");
+    private static final Pattern TYPE_TAG = Pattern.compile("^ Type is\\s.*");
 
     /** Regular expression for property validation type location in class-level javadocs. */
     private static final Pattern VALIDATION_TYPE_TAG =
             Pattern.compile("\\s.*Validation type is\\s.*");
 
     /** Regular expression for property default value location in class-level javadocs. */
-    private static final Pattern DEFAULT_VALUE_TAG = Pattern.compile("\\s*Default value is:*.*");
+    private static final Pattern DEFAULT_VALUE_TAG = Pattern.compile("^ Default value is:*.*");
 
     /** Regular expression for check example location in class-level javadocs. */
     private static final Pattern EXAMPLES_TAG =
@@ -104,6 +105,17 @@ public class JavadocMetadataScraper extends AbstractJavadocCheck {
                         + "config.html#Checker>Checker</a> module"
     )));
 
+    /**
+     * Format for exception message for missing type for check property.
+     */
+    private static final String PROP_TYPE_MISSING = "Type for property '%s' is missing";
+
+    /**
+     * Format for exception message for missing default value for check property.
+     */
+    private static final String PROP_DEFAULT_VALUE_MISSING =
+        "Default value for property '%s' is missing";
+
     /** ModuleDetails instance for each module AST traversal. */
     private ModuleDetails moduleDetails;
 
@@ -261,38 +273,41 @@ private static ModulePropertyDetails createProperties(DetailNode nodeLi) {
                 JavadocTokenTypes.JAVADOC_INLINE_TAG, 0);
         if (propertyNameNode.isPresent()) {
             final DetailNode propertyNameTag = propertyNameNode.get();
+            final String propertyName = getTextFromTag(propertyNameTag);
 
-            final Optional<DetailNode> propertyTypeNode =
-                    getFirstChildOfMatchingText(nodeLi, TYPE_TAG);
-            if (propertyTypeNode.isPresent()) {
-                final DetailNode propertyType = propertyTypeNode.get();
-                final String propertyDesc = DESC_CLEAN.matcher(
-                        constructSubTreeText(nodeLi, propertyNameTag.getIndex() + 1,
-                                propertyType.getIndex() - 1))
-                        .replaceAll(Matcher.quoteReplacement(""));
-
-                modulePropertyDetails.setDescription(propertyDesc.trim());
-            }
-
-            modulePropertyDetails.setName(getTextFromTag(propertyNameTag));
-        }
+            final DetailNode propertyType = getFirstChildOfMatchingText(nodeLi, TYPE_TAG)
+                .orElseThrow(() -> {
+                    return new MetadataGenerationException(String.format(
+                        Locale.ROOT, PROP_TYPE_MISSING, propertyName)
+                    );
+                });
+            final String propertyDesc = DESC_CLEAN.matcher(
+                    constructSubTreeText(nodeLi, propertyNameTag.getIndex() + 1,
+                            propertyType.getIndex() - 1))
+                    .replaceAll(Matcher.quoteReplacement(""));
 
-        final Optional<DetailNode> typeNode = getFirstChildOfMatchingText(nodeLi, TYPE_TAG);
-        if (typeNode.isPresent()) {
-            modulePropertyDetails.setType(getTagTextFromProperty(nodeLi, typeNode.get()));
-        }
+            modulePropertyDetails.setDescription(propertyDesc.trim());
+            modulePropertyDetails.setName(propertyName);
+            modulePropertyDetails.setType(getTagTextFromProperty(nodeLi, propertyType));
 
-        final Optional<DetailNode> validationTypeNodeOpt = getFirstChildOfMatchingText(nodeLi,
+            final Optional<DetailNode> validationTypeNodeOpt = getFirstChildOfMatchingText(nodeLi,
                 VALIDATION_TYPE_TAG);
-        if (validationTypeNodeOpt.isPresent()) {
-            final DetailNode validationTypeNode = validationTypeNodeOpt.get();
-            modulePropertyDetails.setValidationType(getTagTextFromProperty(nodeLi,
+            if (validationTypeNodeOpt.isPresent()) {
+                final DetailNode validationTypeNode = validationTypeNodeOpt.get();
+                modulePropertyDetails.setValidationType(getTagTextFromProperty(nodeLi,
                     validationTypeNode));
-        }
+            }
 
-        final String defaultValue = getPropertyDefaultText(nodeLi);
-        if (!PROPERTIES_TO_NOT_WRITE.contains(defaultValue)) {
-            modulePropertyDetails.setDefaultValue(defaultValue);
+            final String defaultValue = getFirstChildOfMatchingText(nodeLi, DEFAULT_VALUE_TAG)
+                .map(defaultValueNode -> getPropertyDefaultText(nodeLi, defaultValueNode))
+                .orElseThrow(() -> {
+                    return new MetadataGenerationException(String.format(
+                        Locale.ROOT, PROP_DEFAULT_VALUE_MISSING, propertyName)
+                    );
+                });
+            if (!PROPERTIES_TO_NOT_WRITE.contains(defaultValue)) {
+                modulePropertyDetails.setDefaultValue(defaultValue);
+            }
         }
         return modulePropertyDetails;
     }
@@ -398,28 +413,24 @@ else if (exampleSectionStartIdx > -1) {
      * Create property default text, which is either normal property value or list of tokens.
      *
      * @param nodeLi list item javadoc node
+     * @param defaultValueNode default value node
      * @return default property text
      */
-    private static String getPropertyDefaultText(DetailNode nodeLi) {
-        String result = "";
-        final Optional<DetailNode> defaultValueNodeOpt = getFirstChildOfMatchingText(nodeLi,
-                DEFAULT_VALUE_TAG);
-        if (defaultValueNodeOpt.isPresent()) {
-            final DetailNode defaultValueNode = defaultValueNodeOpt.get();
-            final Optional<DetailNode> propertyDefaultValueTagNode = getFirstChildOfType(nodeLi,
-                    JavadocTokenTypes.JAVADOC_INLINE_TAG, defaultValueNode.getIndex() + 1);
-            DetailNode propertyDefaultValueTag = null;
-            if (propertyDefaultValueTagNode.isPresent()) {
-                propertyDefaultValueTag = propertyDefaultValueTagNode.get();
-            }
-            if (propertyDefaultValueTag == null) {
-                final String tokenText = constructSubTreeText(nodeLi,
-                        defaultValueNode.getIndex(), nodeLi.getChildren().length);
-                result = cleanDefaultTokensText(tokenText);
-            }
-            else {
-                result = getTextFromTag(propertyDefaultValueTag);
-            }
+    private static String getPropertyDefaultText(DetailNode nodeLi, DetailNode defaultValueNode) {
+        final Optional<DetailNode> propertyDefaultValueTagNode = getFirstChildOfType(nodeLi,
+                JavadocTokenTypes.JAVADOC_INLINE_TAG, defaultValueNode.getIndex() + 1);
+        DetailNode propertyDefaultValueTag = null;
+        if (propertyDefaultValueTagNode.isPresent()) {
+            propertyDefaultValueTag = propertyDefaultValueTagNode.get();
+        }
+        final String result;
+        if (propertyDefaultValueTag == null) {
+            final String tokenText = constructSubTreeText(nodeLi,
+                    defaultValueNode.getIndex(), nodeLi.getChildren().length);
+            result = cleanDefaultTokensText(tokenText);
+        }
+        else {
+            result = getTextFromTag(propertyDefaultValueTag);
         }
         return result;
     }
diff --git a/src/main/java/com/puppycrawl/tools/checkstyle/meta/MetadataGenerationException.java b/src/main/java/com/puppycrawl/tools/checkstyle/meta/MetadataGenerationException.java
new file mode 100644
--- /dev/null
+++ b/src/main/java/com/puppycrawl/tools/checkstyle/meta/MetadataGenerationException.java
@@ -0,0 +1,49 @@
+////////////////////////////////////////////////////////////////////////////////
+// checkstyle: Checks Java source code for adherence to a set of rules.
+// Copyright (C) 2001-2021 the original author or authors.
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+////////////////////////////////////////////////////////////////////////////////
+
+package com.puppycrawl.tools.checkstyle.meta;
+
+/**
+ * Exception for metadata generation errors.
+ */
+public class MetadataGenerationException extends RuntimeException {
+
+    /** For serialization that will never happen. */
+    private static final long serialVersionUID = -254525356420746283L;
+
+    /**
+     * Creates a new {@code MetadataGenerationException} instance.
+     *
+     * @param message a {@code String} value
+     */
+    public MetadataGenerationException(String message) {
+        super(message);
+    }
+
+    /**
+     * Creates a new {@code MetadataGenerationException} instance
+     * that was caused by another exception.
+     *
+     * @param message a message that explains this exception
+     * @param cause the Exception that is wrapped by this exception
+     */
+    public MetadataGenerationException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}
diff --git a/src/main/java/com/puppycrawl/tools/checkstyle/meta/MetadataGeneratorUtil.java b/src/main/java/com/puppycrawl/tools/checkstyle/meta/MetadataGeneratorUtil.java
--- a/src/main/java/com/puppycrawl/tools/checkstyle/meta/MetadataGeneratorUtil.java
+++ b/src/main/java/com/puppycrawl/tools/checkstyle/meta/MetadataGeneratorUtil.java
@@ -35,7 +35,6 @@
 import com.puppycrawl.tools.checkstyle.DefaultConfiguration;
 import com.puppycrawl.tools.checkstyle.TreeWalker;
 import com.puppycrawl.tools.checkstyle.api.CheckstyleException;
-import com.puppycrawl.tools.checkstyle.utils.TokenUtil;
 
 /** Class which handles all the metadata generation and writing calls. */
 public final class MetadataGeneratorUtil {
@@ -99,15 +98,4 @@ private static void dumpMetadata(Checker checker, String path) throws Checkstyle
 
         checker.process(validFiles);
     }
-
-    /**
-     * Return all token types present in checkstyle.
-     *
-     * @return list of token type names
-     */
-    public static List<String> fetchAllTokens() {
-        return Arrays.stream(TokenUtil.getAllTokenIds())
-                .mapToObj(TokenUtil::getTokenName)
-                .collect(Collectors.toList());
-    }
 }


Test patch:
diff --git a/src/test/java/com/puppycrawl/tools/checkstyle/meta/MetadataGeneratorUtilTest.java b/src/test/java/com/puppycrawl/tools/checkstyle/meta/MetadataGeneratorUtilTest.java
--- a/src/test/java/com/puppycrawl/tools/checkstyle/meta/MetadataGeneratorUtilTest.java
+++ b/src/test/java/com/puppycrawl/tools/checkstyle/meta/MetadataGeneratorUtilTest.java
@@ -19,7 +19,9 @@
 
 package com.puppycrawl.tools.checkstyle.meta;
 
+import static com.google.common.truth.Truth.assertThat;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
 
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -33,9 +35,15 @@
 
 import org.junit.jupiter.api.Test;
 
+import com.puppycrawl.tools.checkstyle.api.CheckstyleException;
 import com.puppycrawl.tools.checkstyle.internal.utils.CheckUtil;
 
 public final class MetadataGeneratorUtilTest {
+
+    private final String invalidMetadataPackage = System.getProperty("user.dir")
+        + "/src/test/resources/com/puppycrawl/tools/checkstyle"
+        + "/meta/javadocmetadatascraper/invalid_metadata";
+
     private final List<String> modulesContainingNoMetadataFile = Arrays.asList(
             "Checker",
             "TreeWalker",
@@ -43,7 +51,7 @@ public final class MetadataGeneratorUtilTest {
     );
 
     @Test
-    public void generateMetadataFiles() throws Exception {
+    public void testMetadataFilesGenerationAllFiles() throws Exception {
         MetadataGeneratorUtil.generate(System.getProperty("user.dir")
                 + "/src/main/java/com/puppycrawl/tools/checkstyle");
         final Set<String> metaFiles;
@@ -67,6 +75,50 @@ public void generateMetadataFiles() throws Exception {
                         + "module", checkstyleModules, metaFiles);
     }
 
+    @Test
+    public void testMetadataFileGenerationDefaultValueMisplaced() {
+        final CheckstyleException exc = assertThrows(CheckstyleException.class, () -> {
+            MetadataGeneratorUtil.generate(invalidMetadataPackage
+                + "/InputJavadocMetadataScraperPropertyMisplacedDefaultValueCheck.java");
+        });
+        assertThat(exc.getCause()).isInstanceOf(MetadataGenerationException.class);
+        assertThat(exc.getCause().getMessage()).isEqualTo(
+            "Default value for property 'misplacedDefaultValue' is missing");
+    }
+
+    @Test
+    public void testMetadataFileGenerationTypeMisplaced() {
+        final CheckstyleException exc = assertThrows(CheckstyleException.class, () -> {
+            MetadataGeneratorUtil.generate(invalidMetadataPackage
+                + "/InputJavadocMetadataScraperPropertyMisplacedTypeCheck.java");
+        });
+        assertThat(exc.getCause()).isInstanceOf(MetadataGenerationException.class);
+        assertThat(exc.getCause().getMessage()).isEqualTo(
+            "Type for property 'misplacedType' is missing");
+    }
+
+    @Test
+    public void testMetadataFileGenerationTypeMissing() {
+        final CheckstyleException exc = assertThrows(CheckstyleException.class, () -> {
+            MetadataGeneratorUtil.generate(invalidMetadataPackage
+                + "/InputJavadocMetadataScraperPropertyMissingTypeCheck.java");
+        });
+        assertThat(exc.getCause()).isInstanceOf(MetadataGenerationException.class);
+        assertThat(exc.getCause().getMessage()).isEqualTo(
+            "Type for property 'missingType' is missing");
+    }
+
+    @Test
+    public void testMetadataFileGenerationDefaultValueMissing() {
+        final CheckstyleException exc = assertThrows(CheckstyleException.class, () -> {
+            MetadataGeneratorUtil.generate(invalidMetadataPackage
+                + "/InputJavadocMetadataScraperPropertyMissingDefaultValueCheck.java");
+        });
+        assertThat(exc.getCause()).isInstanceOf(MetadataGenerationException.class);
+        assertThat(exc.getCause().getMessage()).isEqualTo(
+            "Default value for property 'missingDefaultValue' is missing");
+    }
+
     /**
      * Get meta file name from full file name.
      *
diff --git a/src/test/resources/com/puppycrawl/tools/checkstyle/meta/javadocmetadatascraper/invalid_metadata/InputJavadocMetadataScraperPropertyMisplacedDefaultValueCheck.java b/src/test/resources/com/puppycrawl/tools/checkstyle/meta/javadocmetadatascraper/invalid_metadata/InputJavadocMetadataScraperPropertyMisplacedDefaultValueCheck.java
new file mode 100644
--- /dev/null
+++ b/src/test/resources/com/puppycrawl/tools/checkstyle/meta/javadocmetadatascraper/invalid_metadata/InputJavadocMetadataScraperPropertyMisplacedDefaultValueCheck.java
@@ -0,0 +1,17 @@
+package com.puppycrawl.tools.checkstyle.meta.javadocmetadatascraper.invalid_metadata;
+
+/**
+ * <p>
+ * Description.
+ * </p>
+ * <ul>
+ * <li>
+ * Property {@code misplacedDefaultValue} - Some description.
+ * Type is {@code java.lang.String}. Default value is {@code ""}.
+ * </li>
+ * </ul>
+ *
+ * @since 8.33
+ */
+public class InputJavadocMetadataScraperPropertyMisplacedDefaultValueCheck {
+}
diff --git a/src/test/resources/com/puppycrawl/tools/checkstyle/meta/javadocmetadatascraper/invalid_metadata/InputJavadocMetadataScraperPropertyMisplacedTypeCheck.java b/src/test/resources/com/puppycrawl/tools/checkstyle/meta/javadocmetadatascraper/invalid_metadata/InputJavadocMetadataScraperPropertyMisplacedTypeCheck.java
new file mode 100644
--- /dev/null
+++ b/src/test/resources/com/puppycrawl/tools/checkstyle/meta/javadocmetadatascraper/invalid_metadata/InputJavadocMetadataScraperPropertyMisplacedTypeCheck.java
@@ -0,0 +1,30 @@
+package com.puppycrawl.tools.checkstyle.meta.javadocmetadatascraper.invalid_metadata;
+
+/**
+ * <p>
+ * Checks whether file contains code. Files which are considered to have no code:
+ * </p>
+ * <ul>
+ * <li>
+ * File with no text
+ * </li>
+ * <li>
+ * File with single line comment(s)
+ * </li>
+ * <li>
+ * File with a multi line comment(s).
+ * </li>
+ * </ul>
+ * <p>
+ * <ul>
+ * <li>
+ * Property {@code misplacedType} - Some long wrapped
+ * description. Type is {@code java.lang.String}.
+ * Default value is {@code ""}.
+ * </li>
+ * </ul>
+ *
+ * @since 8.33
+ */
+public class InputJavadocMetadataScraperPropertyMisplacedTypeCheck {
+}
diff --git a/src/test/resources/com/puppycrawl/tools/checkstyle/meta/javadocmetadatascraper/invalid_metadata/InputJavadocMetadataScraperPropertyMissingDefaultValueCheck.java b/src/test/resources/com/puppycrawl/tools/checkstyle/meta/javadocmetadatascraper/invalid_metadata/InputJavadocMetadataScraperPropertyMissingDefaultValueCheck.java
new file mode 100644
--- /dev/null
+++ b/src/test/resources/com/puppycrawl/tools/checkstyle/meta/javadocmetadatascraper/invalid_metadata/InputJavadocMetadataScraperPropertyMissingDefaultValueCheck.java
@@ -0,0 +1,17 @@
+package com.puppycrawl.tools.checkstyle.meta.javadocmetadatascraper.invalid_metadata;
+
+/**
+ * <p>
+ * Description.
+ * </p>
+ * <ul>
+ * <li>
+ * Property {@code missingDefaultValue} - Some description.
+ * Type is {@code java.lang.String}.
+ * </li>
+ * </ul>
+ *
+ * @since 8.33
+ */
+public class InputJavadocMetadataScraperPropertyMissingDefaultValueCheck {
+}
diff --git a/src/test/resources/com/puppycrawl/tools/checkstyle/meta/javadocmetadatascraper/invalid_metadata/InputJavadocMetadataScraperPropertyMissingTypeCheck.java b/src/test/resources/com/puppycrawl/tools/checkstyle/meta/javadocmetadatascraper/invalid_metadata/InputJavadocMetadataScraperPropertyMissingTypeCheck.java
new file mode 100644
--- /dev/null
+++ b/src/test/resources/com/puppycrawl/tools/checkstyle/meta/javadocmetadatascraper/invalid_metadata/InputJavadocMetadataScraperPropertyMissingTypeCheck.java
@@ -0,0 +1,29 @@
+package com.puppycrawl.tools.checkstyle.meta.javadocmetadatascraper.invalid_metadata;
+
+/**
+ * <p>
+ * Checks whether file contains code. Files which are considered to have no code:
+ * </p>
+ * <ul>
+ * <li>
+ * File with no text
+ * </li>
+ * <li>
+ * File with single line comment(s)
+ * </li>
+ * <li>
+ * File with a multi line comment(s).
+ * </li>
+ * </ul>
+ * <p>
+ * <ul>
+ * <li>
+ * Property {@code missingType} - Some description.
+ * Default value is {@code ""}.
+ * </li>
+ * </ul>
+ *
+ * @since 8.33
+ */
+public class InputJavadocMetadataScraperPropertyMissingTypeCheck {
+}