Repository: oppia/oppia

Issue:
Fix #10383: Migrated comment checker to ITokenChecker
## Overview
<!--
READ ME FIRST:
Please answer *both* questions below and check off every point from the Essential Checklist!
If there is no corresponding issue number, fill in N/A where it says [fill_in_number_here] below in 1.
-->

~~1. This PR fixes or fixes part of #[fill_in_number_here].~~
2. This PR does the following: Migrated comment checker to ITokenChecker

## Essential Checklist

- [ ] The PR title starts with "Fix #bugnum: ", followed by a short, clear summary of the changes. (If this PR fixes part of an issue, prefix the title with "Fix part of #bugnum: ...".)
- [x] The linter/Karma presubmit checks have passed locally on your machine.
- [x] "Allow edits from maintainers" is checked. (See [here](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/allowing-changes-to-a-pull-request-branch-created-from-a-fork) for instructions on how to enable it.)
  - This lets reviewers restart your CircleCI tests for you.
- [x] The PR is made from a branch that's **not** called "develop".

## PR Pointers

- Oppiabot will notify you when you don't add a PR_CHANGELOG label. If you are unable to do so, please @-mention a code owner (who will be in the Reviewers list), or ask on [Gitter](https://gitter.im/oppia/oppia-chat).
- For what code owners will expect, see the [Code Owner's wiki page](https://github.com/oppia/oppia/wiki/Oppia%27s-code-owners-and-checks-to-be-carried-out-by-developers).
- Make sure your PR follows conventions in the [style guide](https://github.com/oppia/oppia/wiki/Coding-style-guide), otherwise this will lead to review delays
- Never force push. If you do, your PR will be closed.



Discussion:

Resolving patch:
diff --git a/scripts/linters/pylint_extensions.py b/scripts/linters/pylint_extensions.py
--- a/scripts/linters/pylint_extensions.py
+++ b/scripts/linters/pylint_extensions.py
@@ -42,7 +42,7 @@
 # the punctuation and capital letter checks will be skipped for that
 # comment or docstring.
 EXCLUDED_PHRASES = [
-    'utf', 'pylint:', 'http://', 'https://', 'scripts/', 'extract_node']
+    'coding:', 'pylint:', 'http://', 'https://', 'scripts/', 'extract_node']
 
 import astroid  # isort:skip  pylint: disable=wrong-import-order, wrong-import-position
 from pylint import checkers  # isort:skip  pylint: disable=wrong-import-order, wrong-import-position
@@ -582,7 +582,7 @@ class DocstringParameterChecker(checkers.BaseChecker):
     not_needed_param_in_docstring = {'self', 'cls'}
     docstring_sections = {'Raises:', 'Returns:', 'Yields:'}
 
-    # Docstring section headers split up into arguments, returns, yields and
+    # Docstring section headers split up into arguments, returns, yields
     # and raises sections signifying that we are currently parsing the
     # corresponding section of that docstring.
     DOCSTRING_SECTION_RETURNS = 'returns'
@@ -1633,7 +1633,7 @@ def visit_binop(self, node):
 class SingleLineCommentChecker(checkers.BaseChecker):
     """Checks if comments follow correct style."""
 
-    __implements__ = interfaces.IRawChecker
+    __implements__ = interfaces.ITokenChecker
     name = 'incorrectly_styled_comment'
     priority = -1
     msgs = {
@@ -1653,100 +1653,105 @@ class SingleLineCommentChecker(checkers.BaseChecker):
             'Please use capital letter to begin the content of comment.'
         )
     }
-
-    def process_module(self, node):
-        """Process a module to ensure that comments follow correct style.
+    options = ((
+        'allowed-comment-prefixes',
+        {
+            'default': ('int', 'str', 'float', 'bool', 'v'),
+            'type': 'csv', 'metavar': '<comma separated list>',
+            'help': 'List of allowed prefixes in a comment.'
+        }
+    ),)
+
+    def _check_space_at_beginning_of_comments(self, line, line_num):
+        """Checks if the comment starts with a space.
 
         Args:
-            node: astroid.scoped_nodes.Function. Node to access module content.
+            line: str. The current line of comment.
+            line_num: int. Line number of the current comment.
         """
+        if re.search(br'^#[^\s].*$', line) and not line.startswith(b'#!'):
+            self.add_message(
+                'no-space-at-beginning', line=line_num)
 
-        allowed_comment_prefixes = ['int', 'str', 'float', 'bool', 'v']
-        in_multi_line_comment = False
-        space_at_beginning_of_comment = True
-        multi_line_indicator = b'"""'
-        file_content = read_from_node(node)
-        file_length = len(file_content)
-
-        for line_num in python_utils.RANGE(file_length):
-            line = file_content[line_num].strip()
-
-            # Single multi-line comment, ignore it.
-            if line.count(multi_line_indicator) == 2:
-                continue
+    def _check_comment_starts_with_capital_letter(self, line, line_num):
+        """Checks if the comment starts with a capital letter.
+        Comments may include a lowercase character at the beginning only if they
+        start with version info or a data type or a variable name e.g.
+        "# next_line is of string type." or "# v2 version does not have
+        ExplorationStats Model." or "# int. The file size, in bytes.".
 
-            # Flip multi-line boolean depending on whether or not we see
-            # the multi-line indicator. Possible for multiline comment to
-            # be somewhere other than the start of a line (e.g. func arg),
-            # so we can't look at start of or end of a line, which is why
-            # the case where two indicators in a single line is handled
-            # separately (i.e. one line comment with multi-line strings).
-            if multi_line_indicator in line:
-                in_multi_line_comment = not in_multi_line_comment
-
-            # Ignore anything inside a multiline comment.
-            if in_multi_line_comment:
-                continue
+        Args:
+            line: str. The current line of comment.
+            line_num: int. Line number of the current comment.
+        """
+        # Check if variable name is used.
+        if line[1:].startswith(b' '):
+            starts_with_underscore = '_' in line.split()[1]
+        else:
+            starts_with_underscore = '_' in line.split()[0]
+
+        # Check if allowed prefix is used.
+        allowed_prefix_is_present = any(
+            line[2:].startswith(word) for word in
+            self.config.allowed_comment_prefixes)
+
+        # Check if comment contains any excluded phrase.
+        excluded_phrase_is_present = any(
+            line[1:].strip().startswith(word) for word in EXCLUDED_PHRASES)
+        if (re.search(br'^# [a-z].*', line) and not (
+                excluded_phrase_is_present or
+                starts_with_underscore or allowed_prefix_is_present)):
+            self.add_message(
+                'no-capital-letter-at-beginning', line=line_num)
 
-            next_line = ''
-            previous_line = ''
-            if line_num + 1 < file_length:
-                next_line = file_content[line_num + 1].strip()
-            if line_num > 0:
-                previous_line = file_content[line_num - 1].strip()
+    def _check_punctuation(self, line, line_num):
+        """Checks if the comment starts with a correct punctuation.
 
-            # Ignore lines which are not comments.
-            if not line.startswith(b'#'):
-                continue
+        Args:
+            line: str. The current line of comment.
+            line_num: int. Line number of the current comment.
+        """
+        excluded_phrase_is_present_at_end = any(
+            word in line for word in EXCLUDED_PHRASES)
+        # Comments must end with the proper punctuation.
+        last_char_is_invalid = line[-1] not in (
+            ALLOWED_TERMINATING_PUNCTUATIONS)
+
+        excluded_phrase_at_beginning_of_line = any(
+            line[1:].startswith(word) for word in EXCLUDED_PHRASES)
+        if (last_char_is_invalid and not (
+                excluded_phrase_is_present_at_end or
+                excluded_phrase_at_beginning_of_line)):
+            self.add_message('invalid-punctuation-used', line=line_num)
 
-            # Check if comment contains any excluded phrase.
-            word_is_present_in_excluded_phrases = any(
-                word in line for word in EXCLUDED_PHRASES)
+    def process_tokens(self, tokens):
+        """Custom pylint checker to ensure that comments follow correct style.
 
-            # Comments may include a lowercase character at beginning
-            # or may not use a punctuation at end if it contains a
-            # excluded phrase e.g. "# coding: utf-8".
-            if word_is_present_in_excluded_phrases:
-                continue
+        Args:
+            tokens: list(Token). Object to access all tokens of a module.
+        """
+        prev_line_num = -1
+        comments_group_list = []
+        comments_index = -1
 
-            if not next_line.startswith(b'#'):
-                # Comments must end with the proper punctuation.
-                last_char_is_invalid = line[-1] not in (
-                    ALLOWED_TERMINATING_PUNCTUATIONS)
-                if last_char_is_invalid:
-                    self.add_message(
-                        'invalid-punctuation-used', line=line_num + 1)
+        for (token_type, _, (line_num, _), _, line) in tokens:
+            if token_type == tokenize.COMMENT and line.strip().startswith('#'):
+                line = line.strip()
 
-            # Comments must start with a space.
-            if re.search(br'^#[^\s].*$', line) and not line.startswith(b'#!'):
-                space_at_beginning_of_comment = False
-                self.add_message(
-                    'no-space-at-beginning', line=line_num + 1)
-
-            # Comments may include a lowercase character at the beginning
-            # only if they start with version info or a data type or.
-            # a variable name e.g. "# next_line is of string type."
-            # or "# v2 version does not have ExplorationStatsModel."
-            # or "# int. The file size, in bytes.".
-            if len(line) > 1 and space_at_beginning_of_comment:
-
-                # Check if variable name is used.
-                underscore_is_present = '_' in line.split()[1]
-                if underscore_is_present:
-                    continue
+                self._check_space_at_beginning_of_comments(line, line_num)
 
-                # Check if allowed prefix is used.
-                allowed_prefix_is_present = any(
-                    line[2:].startswith(word) for word in (
-                        allowed_comment_prefixes))
-                if allowed_prefix_is_present:
-                    continue
+                if prev_line_num + 1 == line_num:
+                    comments_group_list[comments_index].append((line, line_num))
+                else:
+                    comments_group_list.append([(line, line_num)])
+                    comments_index += 1
+                prev_line_num = line_num
 
-            # Comments must start with a capital letter.
-            if not previous_line.startswith(b'#') and (
-                    re.search(br'^# [a-z][A-Za-z]*.*$', line)):
-                self.add_message(
-                    'no-capital-letter-at-beginning', line=line_num + 1)
+        for comments in comments_group_list:
+            # Checks first line of comment.
+            self._check_comment_starts_with_capital_letter(*comments[0])
+            # Checks last line of comment.
+            self._check_punctuation(*comments[-1])
 
 
 class BlankLineBelowFileOverviewChecker(checkers.BaseChecker):


Test patch:
diff --git a/scripts/linters/pylint_extensions_test.py b/scripts/linters/pylint_extensions_test.py
--- a/scripts/linters/pylint_extensions_test.py
+++ b/scripts/linters/pylint_extensions_test.py
@@ -2351,17 +2351,20 @@ def test_invalid_punctuation(self):
 
         with python_utils.open_file(filename, 'w') as tmp:
             tmp.write(
-                u"""# Something/
+                u"""# This is a multiline
+                # comment/
+
+                # Comment.
                 """)
         node_invalid_punctuation.file = filename
         node_invalid_punctuation.path = filename
 
-        self.checker_test_object.checker.process_module(
-            node_invalid_punctuation)
+        self.checker_test_object.checker.process_tokens(
+            utils.tokenize_module(node_invalid_punctuation))
 
         message = testutils.Message(
             msg_id='invalid-punctuation-used',
-            line=1)
+            line=2)
 
         with self.checker_test_object.assertAddsMessages(message):
             temp_file.close()
@@ -2380,8 +2383,8 @@ def test_no_space_at_beginning(self):
         node_no_space_at_beginning.file = filename
         node_no_space_at_beginning.path = filename
 
-        self.checker_test_object.checker.process_module(
-            node_no_space_at_beginning)
+        self.checker_test_object.checker.process_tokens(
+            utils.tokenize_module(node_no_space_at_beginning))
 
         message = testutils.Message(
             msg_id='no-space-at-beginning',
@@ -2399,17 +2402,19 @@ def test_no_capital_letter_at_beginning(self):
 
         with python_utils.open_file(filename, 'w') as tmp:
             tmp.write(
-                u"""# something.
+                u"""# coding: utf-8
+
+                    # something.
                 """)
         node_no_capital_letter_at_beginning.file = filename
         node_no_capital_letter_at_beginning.path = filename
 
-        self.checker_test_object.checker.process_module(
-            node_no_capital_letter_at_beginning)
+        self.checker_test_object.checker.process_tokens(
+            utils.tokenize_module(node_no_capital_letter_at_beginning))
 
         message = testutils.Message(
             msg_id='no-capital-letter-at-beginning',
-            line=1)
+            line=3)
 
         with self.checker_test_object.assertAddsMessages(message):
             temp_file.close()
@@ -2425,12 +2430,13 @@ def test_comment_with_excluded_phrase(self):
             tmp.write(
                 u"""# coding: utf-8
                 # pylint: disable
+                a = 1 + 2  # pylint: disable
                 """)
         node_comment_with_excluded_phrase.file = filename
         node_comment_with_excluded_phrase.path = filename
 
-        self.checker_test_object.checker.process_module(
-            node_comment_with_excluded_phrase)
+        self.checker_test_object.checker.process_tokens(
+            utils.tokenize_module(node_comment_with_excluded_phrase))
 
         with self.checker_test_object.assertNoMessages():
             temp_file.close()
@@ -2444,13 +2450,15 @@ def test_variable_name_in_comment(self):
 
         with python_utils.open_file(filename, 'w') as tmp:
             tmp.write(
-                u"""# variable_name is used.
+                u"""# coding: utf-8
+
+                # variable_name is used.
                 """)
         node_variable_name_in_comment.file = filename
         node_variable_name_in_comment.path = filename
 
-        self.checker_test_object.checker.process_module(
-            node_variable_name_in_comment)
+        self.checker_test_object.checker.process_tokens(
+            utils.tokenize_module(node_variable_name_in_comment))
 
         with self.checker_test_object.assertNoMessages():
             temp_file.close()
@@ -2464,13 +2472,15 @@ def test_comment_with_version_info(self):
 
         with python_utils.open_file(filename, 'w') as tmp:
             tmp.write(
-                u"""# v2 is used.
+                u"""# coding: utf-8
+
+                # v2 is used.
                 """)
         node_comment_with_version_info.file = filename
         node_comment_with_version_info.path = filename
 
-        self.checker_test_object.checker.process_module(
-            node_comment_with_version_info)
+        self.checker_test_object.checker.process_tokens(
+            utils.tokenize_module(node_comment_with_version_info))
 
         with self.checker_test_object.assertNoMessages():
             temp_file.close()
@@ -2484,13 +2494,15 @@ def test_data_type_in_comment(self):
 
         with python_utils.open_file(filename, 'w') as tmp:
             tmp.write(
-                u"""# str. variable is type of str.
+                u"""# coding: utf-8
+
+                # str. variable is type of str.
                 """)
         node_data_type_in_comment.file = filename
         node_data_type_in_comment.path = filename
 
-        self.checker_test_object.checker.process_module(
-            node_data_type_in_comment)
+        self.checker_test_object.checker.process_tokens(
+            utils.tokenize_module(node_data_type_in_comment))
 
         with self.checker_test_object.assertNoMessages():
             temp_file.close()
@@ -2504,7 +2516,7 @@ def test_comment_inside_docstring(self):
 
         with python_utils.open_file(filename, 'w') as tmp:
             tmp.write(
-                u"""
+                u"""# coding: utf-8
                     \"\"\"# str. variable is type of str.\"\"\"
                     \"\"\"# str. variable is type
                     of str.\"\"\"
@@ -2512,8 +2524,8 @@ def test_comment_inside_docstring(self):
         node_comment_inside_docstring.file = filename
         node_comment_inside_docstring.path = filename
 
-        self.checker_test_object.checker.process_module(
-            node_comment_inside_docstring)
+        self.checker_test_object.checker.process_tokens(
+            utils.tokenize_module(node_comment_inside_docstring))
 
         with self.checker_test_object.assertNoMessages():
             temp_file.close()
@@ -2527,15 +2539,17 @@ def test_well_formed_comment(self):
 
         with python_utils.open_file(filename, 'w') as tmp:
             tmp.write(
-                u"""# Multi
-                    # line
-                    # comment.
+                u"""# coding: utf-8
+
+                # Multi
+                # line
+                # comment.
                 """)
         node_with_no_error_message.file = filename
         node_with_no_error_message.path = filename
 
-        self.checker_test_object.checker.process_module(
-            node_with_no_error_message)
+        self.checker_test_object.checker.process_tokens(
+            utils.tokenize_module(node_with_no_error_message))
 
         with self.checker_test_object.assertNoMessages():
             temp_file.close()
@@ -2588,6 +2602,7 @@ def test_extra_empty_lines_below_fileoverview(self):
         with python_utils.open_file(filename, 'w') as tmp:
             tmp.write(
                 u"""
+
                     \"\"\" this file does something \"\"\"