Repository: avast/yaramod (language: python)

Issue:
Autoformatting does not separate includes and imports with blank line
Autoformatting something like this 

```yara
include "file.yar" import "cuckoo"
```

does not do anything. I would expect autoformatter to separate includes and imports with blank line.

```yara
include "file.yar"

import "cuckoo"
```

Thank you.


Discussion / hints:

Fix patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fe558e19..9063dbf3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,7 @@
 # Changelog
 
+* Separate includes and imports with blank lines ([#187](https://github.com/avast/yaramod/pull/187), [#130](https://github.com/avast/yaramod/issues/130))
+
 # v3.11.0 (2021-08-30)
 
 * Added additional constants of enums to `pe` module ([#183](https://github.com/avast/yaramod/pull/183))
diff --git a/docs/rtd/requirements.txt b/docs/rtd/requirements.txt
index f5115b6f..da6a951d 100644
--- a/docs/rtd/requirements.txt
+++ b/docs/rtd/requirements.txt
@@ -1,3 +1,3 @@
-Sphinx>=2.4.2,<2.5.0
-sphinx_rtd_theme>=0.4.3,<0.5.0
-sphinx-tabs>=1.1.13,<1.2.0
+Sphinx>=4.3.0
+sphinx_rtd_theme>=0.4.3
+sphinx-tabs>=1.1.13
diff --git a/src/types/token_stream.cpp b/src/types/token_stream.cpp
index 5fc78837..3a0ae749 100644
--- a/src/types/token_stream.cpp
+++ b/src/types/token_stream.cpp
@@ -494,6 +494,7 @@ void TokenStream::addMissingNewLines()
 {
 	BracketStack brackets;
 	std::size_t lineCounter = 0;
+	bool comment_after_include = false;
 	for (auto it = begin(); it != end(); ++it)
 	{
 		auto current = it->getType();
@@ -515,6 +516,33 @@ void TokenStream::addMissingNewLines()
 				next = nextIt->getType();
 			}
 		}
+		if (current == TokenType::INCLUDE_PATH || comment_after_include)
+		{
+			comment_after_include = false;
+			auto nextNextIt = std::next(nextIt);
+			auto nextNext = nextNextIt->getType();
+			if (nextNextIt == end())
+				continue;
+			if (next == TokenType::NEW_LINE)
+			{
+				if (nextNext != TokenType::NEW_LINE && nextNext != TokenType::INCLUDE_DIRECTIVE && nextNext != TokenType::COMMENT && nextNext != TokenType::ONELINE_COMMENT)
+				{
+					nextIt = emplace(nextIt, TokenType::NEW_LINE, _new_line_style);
+					next = nextIt->getType();
+				}
+			}
+			else if (next == TokenType::COMMENT || next == TokenType::ONELINE_COMMENT)
+			{
+				comment_after_include = true;
+			}
+			else
+			{
+				if (next != TokenType::NEW_LINE && next != TokenType::INCLUDE_DIRECTIVE && next != TokenType::COMMENT && next != TokenType::ONELINE_COMMENT )
+					nextIt = emplace(nextIt, TokenType::NEW_LINE, _new_line_style);
+				nextIt = emplace(nextIt, TokenType::NEW_LINE, _new_line_style);
+				next = nextIt->getType();
+			}
+		}
 		if (next == TokenType::RP || next == TokenType::RP_ENUMERATION || next == TokenType::RSQB_ENUMERATION || next == TokenType::HEX_JUMP_RIGHT_BRACKET || next == TokenType::REGEXP_END_SLASH || next == TokenType::HEX_END_BRACKET || next == TokenType::RP_WITH_SPACE_BEFORE || next == TokenType::RP_WITH_SPACES)
 		{
 			if (brackets.putNewlineInCurrentSector() && current != TokenType::NEW_LINE)


Test patch:
diff --git a/tests/python/test_parser.py b/tests/python/test_parser.py
index 00325ed2..d65ac7bb 100644
--- a/tests/python/test_parser.py
+++ b/tests/python/test_parser.py
@@ -1876,8 +1876,9 @@ def test_include_file_2(self):
 }''', yara_file.text)
 
     def test_include_undefined_file_in_incomplete_mode(self):
-        yara_file = yaramod.Yaramod().parse_string(parser_mode=yaramod.ParserMode.Incomplete, str=r'''include "nonexistent.yar"
-
+        yara_file = yaramod.Yaramod().parse_string(parser_mode=yaramod.ParserMode.Incomplete, str=r'''include "nonexistent1.yar"
+include "nonexistent2.yar" include "nonexistent3.yar" /* comment 1 */ include "nonexistent4.yar" // comment 2
+include "nonexistent5.yar" // comment 3
 rule rule1 : Tag1 {
     strings:
         $1 = "Hello World!"
@@ -1898,8 +1899,53 @@ def test_include_undefined_file_in_incomplete_mode(self):
 		false
 }''', yara_file.text)
 
+        expected = r'''include "nonexistent1.yar"
+include "nonexistent2.yar"
+include "nonexistent3.yar" /* comment 1 */
+include "nonexistent4.yar" // comment 2
+include "nonexistent5.yar" // comment 3
+
+rule rule1 : Tag1
+{
+	strings:
+		$1 = "Hello World!"
+		$2 = { 01 23 45 67 89 AB CD EF }
+		$3 = /def/is
+	condition:
+		false
+}
+'''
+        self.assertEqual(expected, yara_file.text_formatted)
+
+    def test_include_undefined_file_and_import_in_incomplete_mode(self):
+        yara_file = yaramod.Yaramod().parse_string(parser_mode=yaramod.ParserMode.Incomplete, str=r'''include "nonexistent.yar" import "cuckoo"
+
+rule rule1 : Tag1 {
+    strings:
+        $1 = "Hello World!"
+        $2 = { 01 23 45 67 89 AB CD EF }
+        $3 = /def/is
+    condition:
+        false
+}
+''')
+        rule = yara_file.rules[0]
+
+        self.assertEqual('''import "cuckoo"
+
+rule rule1 : Tag1 {
+	strings:
+		$1 = "Hello World!"
+		$2 = { 01 23 45 67 89 AB CD EF }
+		$3 = /def/is
+	condition:
+		false
+}''', yara_file.text)
+
         expected = r'''include "nonexistent.yar"
 
+import "cuckoo"
+
 rule rule1 : Tag1
 {
 	strings:
@@ -1909,5 +1955,34 @@ def test_include_undefined_file_in_incomplete_mode(self):
 	condition:
 		false
 }
+'''
+        self.assertEqual(expected, yara_file.text_formatted)
+
+    def test_include_file_and_import_in_regular_mode(self):
+        yara_file = yaramod.Yaramod().parse_file('./tests/python/testing_rules/testing_file_with_import.yar')
+        rule = yara_file.rules[0]
+
+        self.assertEqual('''import "cuckoo"
+
+rule RULE {
+	condition:
+		true
+}
+
+rule rule1 {
+	condition:
+		RULE and true
+}''', yara_file.text)
+
+        expected = r'''include "testing_include.yar"
+
+import "cuckoo"
+
+rule rule1
+{
+	condition:
+		RULE and
+		true
+}
 '''
         self.assertEqual(expected, yara_file.text_formatted)
diff --git a/tests/python/testing_rules/testing_file_with_import.yar b/tests/python/testing_rules/testing_file_with_import.yar
new file mode 100644
index 00000000..0acf277e
--- /dev/null
+++ b/tests/python/testing_rules/testing_file_with_import.yar
@@ -0,0 +1,8 @@
+include "testing_include.yar"
+import "cuckoo"
+
+rule rule1
+{
+	condition:
+		RULE and true
+}