|
import hashlib |
|
import os |
|
from typing import Generic, TypeVar, Union, Dict, Optional, Any |
|
from pathlib import Path |
|
|
|
from parso._compatibility import is_pypy |
|
from parso.pgen2 import generate_grammar |
|
from parso.utils import split_lines, python_bytes_to_unicode, \ |
|
PythonVersionInfo, parse_version_string |
|
from parso.python.diff import DiffParser |
|
from parso.python.tokenize import tokenize_lines, tokenize |
|
from parso.python.token import PythonTokenTypes |
|
from parso.cache import parser_cache, load_module, try_to_save_module |
|
from parso.parser import BaseParser |
|
from parso.python.parser import Parser as PythonParser |
|
from parso.python.errors import ErrorFinderConfig |
|
from parso.python import pep8 |
|
from parso.file_io import FileIO, KnownContentFileIO |
|
from parso.normalizer import RefactoringNormalizer, NormalizerConfig |
|
|
|
_loaded_grammars: Dict[str, 'Grammar'] = {} |
|
|
|
_NodeT = TypeVar("_NodeT") |
|
|
|
|
|
class Grammar(Generic[_NodeT]): |
|
""" |
|
:py:func:`parso.load_grammar` returns instances of this class. |
|
|
|
Creating custom none-python grammars by calling this is not supported, yet. |
|
|
|
:param text: A BNF representation of your grammar. |
|
""" |
|
_start_nonterminal: str |
|
_error_normalizer_config: Optional[ErrorFinderConfig] = None |
|
_token_namespace: Any = None |
|
_default_normalizer_config: NormalizerConfig = pep8.PEP8NormalizerConfig() |
|
|
|
def __init__(self, text: str, *, tokenizer, parser=BaseParser, diff_parser=None): |
|
self._pgen_grammar = generate_grammar( |
|
text, |
|
token_namespace=self._get_token_namespace() |
|
) |
|
self._parser = parser |
|
self._tokenizer = tokenizer |
|
self._diff_parser = diff_parser |
|
self._hashed = hashlib.sha256(text.encode("utf-8")).hexdigest() |
|
|
|
def parse(self, |
|
code: Union[str, bytes] = None, |
|
*, |
|
error_recovery=True, |
|
path: Union[os.PathLike, str] = None, |
|
start_symbol: str = None, |
|
cache=False, |
|
diff_cache=False, |
|
cache_path: Union[os.PathLike, str] = None, |
|
file_io: FileIO = None) -> _NodeT: |
|
""" |
|
If you want to parse a Python file you want to start here, most likely. |
|
|
|
If you need finer grained control over the parsed instance, there will be |
|
other ways to access it. |
|
|
|
:param str code: A unicode or bytes string. When it's not possible to |
|
decode bytes to a string, returns a |
|
:py:class:`UnicodeDecodeError`. |
|
:param bool error_recovery: If enabled, any code will be returned. If |
|
it is invalid, it will be returned as an error node. If disabled, |
|
you will get a ParseError when encountering syntax errors in your |
|
code. |
|
:param str start_symbol: The grammar rule (nonterminal) that you want |
|
to parse. Only allowed to be used when error_recovery is False. |
|
:param str path: The path to the file you want to open. Only needed for caching. |
|
:param bool cache: Keeps a copy of the parser tree in RAM and on disk |
|
if a path is given. Returns the cached trees if the corresponding |
|
files on disk have not changed. Note that this stores pickle files |
|
on your file system (e.g. for Linux in ``~/.cache/parso/``). |
|
:param bool diff_cache: Diffs the cached python module against the new |
|
code and tries to parse only the parts that have changed. Returns |
|
the same (changed) module that is found in cache. Using this option |
|
requires you to not do anything anymore with the cached modules |
|
under that path, because the contents of it might change. This |
|
option is still somewhat experimental. If you want stability, |
|
please don't use it. |
|
:param bool cache_path: If given saves the parso cache in this |
|
directory. If not given, defaults to the default cache places on |
|
each platform. |
|
|
|
:return: A subclass of :py:class:`parso.tree.NodeOrLeaf`. Typically a |
|
:py:class:`parso.python.tree.Module`. |
|
""" |
|
if code is None and path is None and file_io is None: |
|
raise TypeError("Please provide either code or a path.") |
|
|
|
if isinstance(path, str): |
|
path = Path(path) |
|
if isinstance(cache_path, str): |
|
cache_path = Path(cache_path) |
|
|
|
if start_symbol is None: |
|
start_symbol = self._start_nonterminal |
|
|
|
if error_recovery and start_symbol != 'file_input': |
|
raise NotImplementedError("This is currently not implemented.") |
|
|
|
if file_io is None: |
|
if code is None: |
|
file_io = FileIO(path) |
|
else: |
|
file_io = KnownContentFileIO(path, code) |
|
|
|
if cache and file_io.path is not None: |
|
module_node = load_module(self._hashed, file_io, cache_path=cache_path) |
|
if module_node is not None: |
|
return module_node |
|
|
|
if code is None: |
|
code = file_io.read() |
|
code = python_bytes_to_unicode(code) |
|
|
|
lines = split_lines(code, keepends=True) |
|
if diff_cache: |
|
if self._diff_parser is None: |
|
raise TypeError("You have to define a diff parser to be able " |
|
"to use this option.") |
|
try: |
|
module_cache_item = parser_cache[self._hashed][file_io.path] |
|
except KeyError: |
|
pass |
|
else: |
|
module_node = module_cache_item.node |
|
old_lines = module_cache_item.lines |
|
if old_lines == lines: |
|
return module_node |
|
|
|
new_node = self._diff_parser( |
|
self._pgen_grammar, self._tokenizer, module_node |
|
).update( |
|
old_lines=old_lines, |
|
new_lines=lines |
|
) |
|
try_to_save_module(self._hashed, file_io, new_node, lines, |
|
|
|
pickling=cache and not is_pypy, |
|
cache_path=cache_path) |
|
return new_node |
|
|
|
tokens = self._tokenizer(lines) |
|
|
|
p = self._parser( |
|
self._pgen_grammar, |
|
error_recovery=error_recovery, |
|
start_nonterminal=start_symbol |
|
) |
|
root_node = p.parse(tokens=tokens) |
|
|
|
if cache or diff_cache: |
|
try_to_save_module(self._hashed, file_io, root_node, lines, |
|
|
|
pickling=cache and not is_pypy, |
|
cache_path=cache_path) |
|
return root_node |
|
|
|
def _get_token_namespace(self): |
|
ns = self._token_namespace |
|
if ns is None: |
|
raise ValueError("The token namespace should be set.") |
|
return ns |
|
|
|
def iter_errors(self, node): |
|
""" |
|
Given a :py:class:`parso.tree.NodeOrLeaf` returns a generator of |
|
:py:class:`parso.normalizer.Issue` objects. For Python this is |
|
a list of syntax/indentation errors. |
|
""" |
|
if self._error_normalizer_config is None: |
|
raise ValueError("No error normalizer specified for this grammar.") |
|
|
|
return self._get_normalizer_issues(node, self._error_normalizer_config) |
|
|
|
def refactor(self, base_node, node_to_str_map): |
|
return RefactoringNormalizer(node_to_str_map).walk(base_node) |
|
|
|
def _get_normalizer(self, normalizer_config): |
|
if normalizer_config is None: |
|
normalizer_config = self._default_normalizer_config |
|
if normalizer_config is None: |
|
raise ValueError("You need to specify a normalizer, because " |
|
"there's no default normalizer for this tree.") |
|
return normalizer_config.create_normalizer(self) |
|
|
|
def _normalize(self, node, normalizer_config=None): |
|
""" |
|
TODO this is not public, yet. |
|
The returned code will be normalized, e.g. PEP8 for Python. |
|
""" |
|
normalizer = self._get_normalizer(normalizer_config) |
|
return normalizer.walk(node) |
|
|
|
def _get_normalizer_issues(self, node, normalizer_config=None): |
|
normalizer = self._get_normalizer(normalizer_config) |
|
normalizer.walk(node) |
|
return normalizer.issues |
|
|
|
def __repr__(self): |
|
nonterminals = self._pgen_grammar.nonterminal_to_dfas.keys() |
|
txt = ' '.join(list(nonterminals)[:3]) + ' ...' |
|
return '<%s:%s>' % (self.__class__.__name__, txt) |
|
|
|
|
|
class PythonGrammar(Grammar): |
|
_error_normalizer_config = ErrorFinderConfig() |
|
_token_namespace = PythonTokenTypes |
|
_start_nonterminal = 'file_input' |
|
|
|
def __init__(self, version_info: PythonVersionInfo, bnf_text: str): |
|
super().__init__( |
|
bnf_text, |
|
tokenizer=self._tokenize_lines, |
|
parser=PythonParser, |
|
diff_parser=DiffParser |
|
) |
|
self.version_info = version_info |
|
|
|
def _tokenize_lines(self, lines, **kwargs): |
|
return tokenize_lines(lines, version_info=self.version_info, **kwargs) |
|
|
|
def _tokenize(self, code): |
|
|
|
return tokenize(code, version_info=self.version_info) |
|
|
|
|
|
def load_grammar(*, version: str = None, path: str = None): |
|
""" |
|
Loads a :py:class:`parso.Grammar`. The default version is the current Python |
|
version. |
|
|
|
:param str version: A python version string, e.g. ``version='3.8'``. |
|
:param str path: A path to a grammar file |
|
""" |
|
version_info = parse_version_string(version) |
|
|
|
file = path or os.path.join( |
|
'python', |
|
'grammar%s%s.txt' % (version_info.major, version_info.minor) |
|
) |
|
|
|
global _loaded_grammars |
|
path = os.path.join(os.path.dirname(__file__), file) |
|
try: |
|
return _loaded_grammars[path] |
|
except KeyError: |
|
try: |
|
with open(path) as f: |
|
bnf_text = f.read() |
|
|
|
grammar = PythonGrammar(version_info, bnf_text) |
|
return _loaded_grammars.setdefault(path, grammar) |
|
except FileNotFoundError: |
|
message = "Python version %s.%s is currently not supported." % ( |
|
version_info.major, version_info.minor |
|
) |
|
raise NotImplementedError(message) |
|
|
