Repository: roskakori/pygount
Pull request #52 (2020-06-23 22:39:11)

#51 Changed preprocessor statements to count as code.


Patch:
diff --git a/dev-requirements.txt b/dev-requirements.txt
index 72c656a..5e7bf2b 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -1,11 +1,11 @@
 black==19.10b0; python_version >= '3.6'
 chardet==3.0.4
-coverage==5.0.4
-coveralls==1.11.1
-pre-commit==2.2.0; python_version >= '3.6'  # pre-commit 2.1 requires Python 3.6+
-pygments==2.5.2  # Pygments 2.5.0 changed Java's "{}" from Operator to Punctuation, earlier versions would break tests.
-pytest==5.4.1
-pytest-cov==2.8.1
-setuptools==46.1.3
+coverage==5.1
+coveralls==2.0.0
+pre-commit==2.5.1; python_version >= '3.6'  # pre-commit 2.1 requires Python 3.6+
+pygments==2.6.1  # Pygments 2.5.0 changed Java's "{}" from Operator to Punctuation, earlier versions would break tests.
+pytest==5.4.3
+pytest-cov==2.10.0
+setuptools==47.3.1
 twine==3.1.1; python_version >= '3.6'  # twine 3.1 requires Python 3.6+
 wheel==0.34.2
diff --git a/docs/changes.rst b/docs/changes.rst
index 77793a9..edd2c65 100644
--- a/docs/changes.rst
+++ b/docs/changes.rst
@@ -5,9 +5,15 @@ Changes
 
 This chapter describes the changes coming with each new version of pygount.
 
+Version 1.2.1, 2020-06-24
+
+* Changed preprocessor statements to count as code, unlike Pygments which
+  treats them as special comments (contributed by nkr0, issue
+  `#51 <https://github.com/roskakori/pygount/issues/51>`_).
+
 Version 1.2.1, 2020-04-02
 
-* Fixed broken links in README on PyPI my moving the documentation to
+* Fixed broken links in README on PyPI by moving the documentation to
   `ReadTheDocs <https://pygount.readthedocs.io/>`_.
 * Improved API:
 
diff --git a/pygount/analysis.py b/pygount/analysis.py
index 40154a0..770f10c 100644
--- a/pygount/analysis.py
+++ b/pygount/analysis.py
@@ -651,7 +651,12 @@ def _line_parts(lexer: pygments.lexer.Lexer, text: str) -> Generator[Set[str], N
     white_text = " \f\n\r\t" + white_characters(language_id)
     white_words = white_code_words(language_id)
     for token_type, token_text in tokens:
-        if token_type in pygments.token.Comment:
+        # NOTE: Pygments treats preprocessor statements as special comments.
+        is_actual_comment = token_type in pygments.token.Comment and token_type not in (
+            pygments.token.Comment.Preproc,
+            pygments.token.Comment.PreprocFile,
+        )
+        if is_actual_comment:
             line_marks.add("d")  # 'documentation'
         elif token_type in pygments.token.String:
             line_marks.add("s")  # 'string'
diff --git a/pygount/common.py b/pygount/common.py
index 5ba1c51..75d32a6 100644
--- a/pygount/common.py
+++ b/pygount/common.py
@@ -11,7 +11,7 @@ import warnings
 from typing import Generator, List, Optional, Pattern, Sequence, Union
 
 
-__version__ = "1.2.1"
+__version__ = "1.2.2"
 
 
 #: Pseudo pattern to indicate that the remaining pattern are an addition to the default patterns.


Test patch:
diff --git a/tests/test_analysis.py b/tests/test_analysis.py
index 9d19e04..cdbaf25 100644
--- a/tests/test_analysis.py
+++ b/tests/test_analysis.py
@@ -5,6 +5,8 @@ Tests for pygount source code analysis.
 # All rights reserved. Distributed under the BSD License.
 import glob
 import os
+from typing import List, Set
+
 import pytest
 import unittest
 
@@ -90,15 +92,39 @@ class AnalysisTest(unittest.TestCase):
         for token_type, token_text in list(analysis._pythonized_comments(analysis._delined_tokens(python_tokens))):
             assert token_type not in token.String
 
+    @staticmethod
+    def _line_parts(lexer_name: str, source_lines: List[str]) -> List[Set[str]]:
+        lexer = lexers.get_lexer_by_name(lexer_name)
+        source_code = "\n".join(source_lines)
+        return list(analysis._line_parts(lexer, source_code))
+
     def test_can_analyze_python(self):
-        source_code = (
-            '"Some tool."\n' "#!/bin/python\n" "#(C) by me\n" "def x():\n" '    "Some function"\n' '    return "abc"\n'
-        )
-        python_lexer = lexers.get_lexer_by_name("python")
-        actual_line_parts = list(analysis._line_parts(python_lexer, source_code))
+        source_lines = [
+            '"Some tool."',
+            "#!/bin/python",
+            "#(C) by me",
+            "def x():",
+            '    "Some function"',
+            '    return "abc"',
+        ]
+        actual_line_parts = AnalysisTest._line_parts("python", source_lines)
         expected_line_parts = [{"d"}, {"d"}, {"d"}, {"c"}, {"d"}, {"c", "s"}]
         assert actual_line_parts == expected_line_parts
 
+    def test_can_analyze_c(self):
+        source_lines = [
+            "/*",
+            " * The classic hello world for C99.",
+            " */",
+            "#include <stdio.h>",
+            "int main(void) {",
+            '   puts("Hello, World!");',
+            "}",
+        ]
+        actual_line_parts = AnalysisTest._line_parts("c", source_lines)
+        expected_line_parts = [{"d"}, {"d"}, {"d"}, {"c"}, {"c"}, {"c", "s"}, set()]
+        assert actual_line_parts == expected_line_parts
+
 
 class FileAnalysisTest(TempFolderTest):
     def test_can_analyze_encoding_error(self):