Repository: avast/yaramod (language: python)

Issue:
Formatted text should strip the trailing whitespace from the comments
Consider a following example:

```python
import yaramod

ymod = yaramod.Yaramod()
f = ymod.parse_string(
    r"""
rule asd {
    // Comment with a trailing space 
    // Comment with a trailing tab	
    // Comment with a trailing combination 	 
    condition:
        false
}
"""
)

print(f.text_formatted)
print(repr(f.text_formatted))
```

All comments contain trailing whitespace and here is anotated the output:

```
'\nrule asd\n{\n\t\t// Comment with a trailing space \n\t\t// Comment with a trailing tab\t\n\t\t// Comment with a trailing combination \t \n\tcondition:\n\t\tfalse\n}\n'
                                                    ^                                    ^                                             ^
                                                    |                                    |                                             +-- space, tab, space
                                                    |                                    +-- tab
                                                    +-- single space character

```

Just to make sure the is there same file in case it is not clear where the whitespace is from the code preview [test.py.zip](https://github.com/avast/yaramod/files/5761621/test.py.zip).

I can't think of a case where keeping the trailing whitespace is intended, especially in the "formatted_text".


Discussion / hints:

Fix patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 838e3fe4..c7a21552 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,6 @@
 # Changelog
 
+* Strip trailing whitespace off oneline comments ([#149](https://github.com/avast/yaramod/issues/149), [#189](https://github.com/avast/yaramod/pull/189))
 * Fix adding meta to add it after comment of last present meta if present ([#102](https://github.com/avast/yaramod/issues/102), [#188](https://github.com/avast/yaramod/pull/188))
 * Separate includes and imports with blank lines ([#187](https://github.com/avast/yaramod/pull/187), [#130](https://github.com/avast/yaramod/issues/130))
 
diff --git a/include/yaramod/types/literal.h b/include/yaramod/types/literal.h
index 28cd9830..a68b4674 100644
--- a/include/yaramod/types/literal.h
+++ b/include/yaramod/types/literal.h
@@ -14,8 +14,8 @@
 #include <sstream>
 #include <variant>
 
-#include "yaramod/yaramod_error.h"
 #include "yaramod/types/symbol.h"
+#include "yaramod/yaramod_error.h"
 
 namespace yaramod {
 
@@ -93,6 +93,7 @@ class Literal
 	/// @name String representation
 	/// @{
 	void markEscaped() {  _escaped = true; }
+	bool trimWhitespaces();
 	std::string getText(bool pure = false) const;
 	std::string getPureText() const;
 	/// @}
diff --git a/include/yaramod/types/token.h b/include/yaramod/types/token.h
index c74ecc95..1297122a 100644
--- a/include/yaramod/types/token.h
+++ b/include/yaramod/types/token.h
@@ -35,6 +35,8 @@ class Token
 		, _location()
 		, _wanted_column(0)
 	{
+		if (_type == TokenType::ONELINE_COMMENT)
+			_value->trimWhitespaces();
 	}
 
 	Token(TokenType type, Literal&& value)
@@ -43,6 +45,8 @@ class Token
 		, _location()
 		, _wanted_column(0)
 	{
+		if (_type == TokenType::ONELINE_COMMENT)
+			_value->trimWhitespaces();
 	}
 
 	Token(const Token& other) = default;
diff --git a/include/yaramod/utils/utils.h b/include/yaramod/utils/utils.h
index a8739c77..79d58ee3 100644
--- a/include/yaramod/utils/utils.h
+++ b/include/yaramod/utils/utils.h
@@ -20,7 +20,7 @@ std::string unescapeString(std::string_view str);
 bool endsWith(const std::string& str, const std::string& withWhat);
 bool endsWith(const std::string& str, char withWhat);
 
-std::string trim(std::string str);
+std::string trim(std::string str, const std::string& toTrim = " \n\r\t\v");
 
 /**
  * Checks whether string starts with another string or character.
diff --git a/src/types/literal.cpp b/src/types/literal.cpp
index e34922c9..58e6e68c 100644
--- a/src/types/literal.cpp
+++ b/src/types/literal.cpp
@@ -214,6 +214,16 @@ std::string Literal::getFormattedValue() const
 	return _formatted_value.value_or(std::string());
 }
 
+bool Literal::trimWhitespaces()
+{
+	if (isString())
+	{
+		setValue(trim(getString(), " \n\t\v"));
+		return true;
+	}
+	return false;
+}
+
 /**
  * Returns the string representation of the literal in a specified format:
  *
diff --git a/src/utils/utils.cpp b/src/utils/utils.cpp
index 6375a88b..b81ff6fc 100644
--- a/src/utils/utils.cpp
+++ b/src/utils/utils.cpp
@@ -189,11 +189,10 @@ bool endsWith(const std::string& str, char withWhat)
 	return !str.empty() && str.back() == withWhat;
 }
 
-std::string trim(std::string str)
+std::string trim(std::string str, const std::string& toTrim)
 {
 	// Based on
 	// http://www.codeproject.com/Articles/10880/A-trim-implementation-for-std-string
-	const std::string toTrim = " \n\r\t\v";
 	std::string::size_type pos = str.find_last_not_of(toTrim);
 	if (pos != std::string::npos)
 	{


Test patch:
diff --git a/tests/cpp/parser_tests.cpp b/tests/cpp/parser_tests.cpp
index df9b5409..dc90e7ba 100644
--- a/tests/cpp/parser_tests.cpp
+++ b/tests/cpp/parser_tests.cpp
@@ -5099,6 +5099,39 @@ rule nonutf_condition
 	EXPECT_EQ(expected, driver.getParsedFile().getTextFormatted());
 }
 
+TEST_F(ParserTests,
+RemoveTrailingWhitespacesFromComments) {
+	prepareInput(
+R"(
+rule trailing_whitespaces_in_comments_rule
+{
+	meta:
+		author = "Mr. Avastien" // comment with extra tab	
+	strings:
+		$s1 = "text" // comment with extra space 
+	condition:
+		$s1
+}
+)");
+	EXPECT_TRUE(driver.parse(input));
+	ASSERT_EQ(1u, driver.getParsedFile().getRules().size());
+
+	std::string expected =
+R"(
+rule trailing_whitespaces_in_comments_rule
+{
+	meta:
+		author = "Mr. Avastien" // comment with extra tab
+	strings:
+		$s1 = "text" // comment with extra space
+	condition:
+		$s1
+}
+)";
+
+	EXPECT_EQ(expected, driver.getParsedFile().getTextFormatted());
+}
+
 TEST_F(ParserTests,
 AddMetaAfterParse) {
 	prepareInput(