Repository: pwndbg/pwndbg

Issue:
Use lexer.py from pwntools instead of embedding it in our repo
The https://github.com/pwndbg/pwndbg/blob/dev/pwndbg/color/lexer.py file we have is a fork of the pwntools file with the same name: https://github.com/Gallopsled/pwntools/blob/dev/pwnlib/lexer.py

@arcz recently fixed a bug in this lexer for aarch64 in https://github.com/pwndbg/pwndbg/commit/456fdb776e05118d5617966cb66974c95b7bd427 and will upstream it to Pwntools.

After it is done, we should try to remove the file from our repo and just use the lexer.py from pwnlib :).


Discussion:
Sent a patch in https://github.com/Gallopsled/pwntools/pull/2139 but the more I look at it the more this seems an invalid issue.

Resolving patch:
diff --git a/pwndbg/color/lexer.py b/pwndbg/color/lexer.py
deleted file mode 100644
--- a/pwndbg/color/lexer.py
+++ /dev/null
@@ -1,94 +0,0 @@
-from __future__ import annotations
-
-from pygments.lexer import RegexLexer
-from pygments.lexer import include
-from pygments.token import Comment
-from pygments.token import Name
-from pygments.token import Number
-from pygments.token import Punctuation
-from pygments.token import String
-from pygments.token import Text
-
-__all__ = ["PwntoolsLexer"]
-
-
-class PwntoolsLexer(RegexLexer):
-    """
-    Fork from pwntools
-    https://github.com/Gallopsled/pwntools/blob/7860eecf025135380b137dd9df85dd02a2fd1667/pwnlib/lexer.py
-
-    Edit:
-        * Remove Objdump rules
-        * Merge pygments-arm (https://github.com/heia-fr/pygments-arm)
-    """
-
-    name = "PwntoolsLexer"
-    filenames = ["*.s", "*.S", "*.asm"]
-
-    #: optional Comment or Whitespace
-    string = r'"(\\"|[^"])*"'
-    char = r"[\w$.@-]"
-    identifier = r"(?:[a-zA-Z$_]" + char + r"*|\." + char + r"+|or)"
-    number = r"(?:\-?0[xX][a-zA-Z0-9]+|\d+)"
-    memory = r"(?:[\]\[])"
-
-    eol = r"[\r\n]+"
-
-    tokens = {
-        "root": [
-            include("whitespace"),
-            # Label
-            (identifier + ":", Name.Label),
-            (number + ":", Name.Label),
-            # AT&T directive
-            (r"\." + identifier, Name.Attribute, "directive-args"),
-            (r"lock|rep(n?z)?|data\d+", Name.Attribute),
-            # Instructions
-            (identifier, Name.Function, "instruction-args"),
-            (r"[\r\n]+", Text),
-        ],
-        "directive-args": [
-            (identifier, Name.Constant),
-            (string, String),
-            ("@" + identifier, Name.Attribute),
-            (number, Number.Integer),
-            (eol, Text, "#pop"),
-            (r"#.*?$", Comment, "#pop"),
-            include("punctuation"),
-            include("whitespace"),
-        ],
-        "instruction-args": [
-            # Fun things
-            (r"([\]\[]|BYTE|DWORD|PTR|\+|\-|}|{|\^|>>|<<|&)", Text),
-            # Address constants
-            (identifier, Name.Constant),
-            ("=" + identifier, Name.Constant),  # ARM symbol
-            (number, Number.Integer),
-            # Registers
-            ("%" + identifier, Name.Variable),
-            ("$" + identifier, Name.Variable),
-            # Numeric constants
-            ("$" + number, Number.Integer),
-            ("#" + number, Number.Integer),
-            # ARM predefined constants
-            ("#" + identifier, Name.Constant),
-            (r"$'(.|\\')'", String.Char),
-            (eol, Text, "#pop"),
-            include("punctuation"),
-            include("whitespace"),
-        ],
-        "whitespace": [
-            (r"\n", Text),
-            (r"\s+", Text),
-            # Block comments
-            # /* */ (AT&T)
-            (r"/\*.*?\*/", Comment),
-            # Line comments
-            # //    (AArch64)
-            # #     (AT&T)
-            # ;     (NASM/intel, LLVM)
-            # @     (ARM)
-            (r"(//|[#;@]).*$", Comment.Single),
-        ],
-        "punctuation": [(r"[-*,.():]+", Punctuation)],
-    }
diff --git a/pwndbg/color/syntax_highlight.py b/pwndbg/color/syntax_highlight.py
--- a/pwndbg/color/syntax_highlight.py
+++ b/pwndbg/color/syntax_highlight.py
@@ -7,12 +7,12 @@
 import pygments
 import pygments.formatters
 import pygments.lexers
+from pwnlib.lexer import PwntoolsLexer
 
 import pwndbg.gdblib.config
 from pwndbg.color import disable_colors
 from pwndbg.color import message
 from pwndbg.color import theme
-from pwndbg.color.lexer import PwntoolsLexer
 
 pwndbg.gdblib.config.add_param("syntax-highlight", True, "Source code / assembly syntax highlight")
 style = theme.add_param(


Test patch: