Spaces:
Running
Running
# common.py | |
from .core import * | |
from .helpers import delimited_list, any_open_tag, any_close_tag | |
from datetime import datetime | |
# some other useful expressions - using lower-case class name since we are really using this as a namespace | |
class pyparsing_common: | |
"""Here are some common low-level expressions that may be useful in | |
jump-starting parser development: | |
- numeric forms (:class:`integers<integer>`, :class:`reals<real>`, | |
:class:`scientific notation<sci_real>`) | |
- common :class:`programming identifiers<identifier>` | |
- network addresses (:class:`MAC<mac_address>`, | |
:class:`IPv4<ipv4_address>`, :class:`IPv6<ipv6_address>`) | |
- ISO8601 :class:`dates<iso8601_date>` and | |
:class:`datetime<iso8601_datetime>` | |
- :class:`UUID<uuid>` | |
- :class:`comma-separated list<comma_separated_list>` | |
- :class:`url` | |
Parse actions: | |
- :class:`convertToInteger` | |
- :class:`convertToFloat` | |
- :class:`convertToDate` | |
- :class:`convertToDatetime` | |
- :class:`stripHTMLTags` | |
- :class:`upcaseTokens` | |
- :class:`downcaseTokens` | |
Example:: | |
pyparsing_common.number.runTests(''' | |
# any int or real number, returned as the appropriate type | |
100 | |
-100 | |
+100 | |
3.14159 | |
6.02e23 | |
1e-12 | |
''') | |
pyparsing_common.fnumber.runTests(''' | |
# any int or real number, returned as float | |
100 | |
-100 | |
+100 | |
3.14159 | |
6.02e23 | |
1e-12 | |
''') | |
pyparsing_common.hex_integer.runTests(''' | |
# hex numbers | |
100 | |
FF | |
''') | |
pyparsing_common.fraction.runTests(''' | |
# fractions | |
1/2 | |
-3/4 | |
''') | |
pyparsing_common.mixed_integer.runTests(''' | |
# mixed fractions | |
1 | |
1/2 | |
-3/4 | |
1-3/4 | |
''') | |
import uuid | |
pyparsing_common.uuid.setParseAction(tokenMap(uuid.UUID)) | |
pyparsing_common.uuid.runTests(''' | |
# uuid | |
12345678-1234-5678-1234-567812345678 | |
''') | |
prints:: | |
# any int or real number, returned as the appropriate type | |
100 | |
[100] | |
-100 | |
[-100] | |
+100 | |
[100] | |
3.14159 | |
[3.14159] | |
6.02e23 | |
[6.02e+23] | |
1e-12 | |
[1e-12] | |
# any int or real number, returned as float | |
100 | |
[100.0] | |
-100 | |
[-100.0] | |
+100 | |
[100.0] | |
3.14159 | |
[3.14159] | |
6.02e23 | |
[6.02e+23] | |
1e-12 | |
[1e-12] | |
# hex numbers | |
100 | |
[256] | |
FF | |
[255] | |
# fractions | |
1/2 | |
[0.5] | |
-3/4 | |
[-0.75] | |
# mixed fractions | |
1 | |
[1] | |
1/2 | |
[0.5] | |
-3/4 | |
[-0.75] | |
1-3/4 | |
[1.75] | |
# uuid | |
12345678-1234-5678-1234-567812345678 | |
[UUID('12345678-1234-5678-1234-567812345678')] | |
""" | |
convert_to_integer = token_map(int) | |
""" | |
Parse action for converting parsed integers to Python int | |
""" | |
convert_to_float = token_map(float) | |
""" | |
Parse action for converting parsed numbers to Python float | |
""" | |
integer = Word(nums).set_name("integer").set_parse_action(convert_to_integer) | |
"""expression that parses an unsigned integer, returns an int""" | |
hex_integer = ( | |
Word(hexnums).set_name("hex integer").set_parse_action(token_map(int, 16)) | |
) | |
"""expression that parses a hexadecimal integer, returns an int""" | |
signed_integer = ( | |
Regex(r"[+-]?\d+") | |
.set_name("signed integer") | |
.set_parse_action(convert_to_integer) | |
) | |
"""expression that parses an integer with optional leading sign, returns an int""" | |
fraction = ( | |
signed_integer().set_parse_action(convert_to_float) | |
+ "/" | |
+ signed_integer().set_parse_action(convert_to_float) | |
).set_name("fraction") | |
"""fractional expression of an integer divided by an integer, returns a float""" | |
fraction.add_parse_action(lambda tt: tt[0] / tt[-1]) | |
mixed_integer = ( | |
fraction | signed_integer + Opt(Opt("-").suppress() + fraction) | |
).set_name("fraction or mixed integer-fraction") | |
"""mixed integer of the form 'integer - fraction', with optional leading integer, returns float""" | |
mixed_integer.add_parse_action(sum) | |
real = ( | |
Regex(r"[+-]?(?:\d+\.\d*|\.\d+)") | |
.set_name("real number") | |
.set_parse_action(convert_to_float) | |
) | |
"""expression that parses a floating point number and returns a float""" | |
sci_real = ( | |
Regex(r"[+-]?(?:\d+(?:[eE][+-]?\d+)|(?:\d+\.\d*|\.\d+)(?:[eE][+-]?\d+)?)") | |
.set_name("real number with scientific notation") | |
.set_parse_action(convert_to_float) | |
) | |
"""expression that parses a floating point number with optional | |
scientific notation and returns a float""" | |
# streamlining this expression makes the docs nicer-looking | |
number = (sci_real | real | signed_integer).setName("number").streamline() | |
"""any numeric expression, returns the corresponding Python type""" | |
fnumber = ( | |
Regex(r"[+-]?\d+\.?\d*([eE][+-]?\d+)?") | |
.set_name("fnumber") | |
.set_parse_action(convert_to_float) | |
) | |
"""any int or real number, returned as float""" | |
identifier = Word(identchars, identbodychars).set_name("identifier") | |
"""typical code identifier (leading alpha or '_', followed by 0 or more alphas, nums, or '_')""" | |
ipv4_address = Regex( | |
r"(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}" | |
).set_name("IPv4 address") | |
"IPv4 address (``0.0.0.0 - 255.255.255.255``)" | |
_ipv6_part = Regex(r"[0-9a-fA-F]{1,4}").set_name("hex_integer") | |
_full_ipv6_address = (_ipv6_part + (":" + _ipv6_part) * 7).set_name( | |
"full IPv6 address" | |
) | |
_short_ipv6_address = ( | |
Opt(_ipv6_part + (":" + _ipv6_part) * (0, 6)) | |
+ "::" | |
+ Opt(_ipv6_part + (":" + _ipv6_part) * (0, 6)) | |
).set_name("short IPv6 address") | |
_short_ipv6_address.add_condition( | |
lambda t: sum(1 for tt in t if pyparsing_common._ipv6_part.matches(tt)) < 8 | |
) | |
_mixed_ipv6_address = ("::ffff:" + ipv4_address).set_name("mixed IPv6 address") | |
ipv6_address = Combine( | |
(_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).set_name( | |
"IPv6 address" | |
) | |
).set_name("IPv6 address") | |
"IPv6 address (long, short, or mixed form)" | |
mac_address = Regex( | |
r"[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}" | |
).set_name("MAC address") | |
"MAC address xx:xx:xx:xx:xx (may also have '-' or '.' delimiters)" | |
def convert_to_date(fmt: str = "%Y-%m-%d"): | |
""" | |
Helper to create a parse action for converting parsed date string to Python datetime.date | |
Params - | |
- fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%d"``) | |
Example:: | |
date_expr = pyparsing_common.iso8601_date.copy() | |
date_expr.setParseAction(pyparsing_common.convertToDate()) | |
print(date_expr.parseString("1999-12-31")) | |
prints:: | |
[datetime.date(1999, 12, 31)] | |
""" | |
def cvt_fn(ss, ll, tt): | |
try: | |
return datetime.strptime(tt[0], fmt).date() | |
except ValueError as ve: | |
raise ParseException(ss, ll, str(ve)) | |
return cvt_fn | |
def convert_to_datetime(fmt: str = "%Y-%m-%dT%H:%M:%S.%f"): | |
"""Helper to create a parse action for converting parsed | |
datetime string to Python datetime.datetime | |
Params - | |
- fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%dT%H:%M:%S.%f"``) | |
Example:: | |
dt_expr = pyparsing_common.iso8601_datetime.copy() | |
dt_expr.setParseAction(pyparsing_common.convertToDatetime()) | |
print(dt_expr.parseString("1999-12-31T23:59:59.999")) | |
prints:: | |
[datetime.datetime(1999, 12, 31, 23, 59, 59, 999000)] | |
""" | |
def cvt_fn(s, l, t): | |
try: | |
return datetime.strptime(t[0], fmt) | |
except ValueError as ve: | |
raise ParseException(s, l, str(ve)) | |
return cvt_fn | |
iso8601_date = Regex( | |
r"(?P<year>\d{4})(?:-(?P<month>\d\d)(?:-(?P<day>\d\d))?)?" | |
).set_name("ISO8601 date") | |
"ISO8601 date (``yyyy-mm-dd``)" | |
iso8601_datetime = Regex( | |
r"(?P<year>\d{4})-(?P<month>\d\d)-(?P<day>\d\d)[T ](?P<hour>\d\d):(?P<minute>\d\d)(:(?P<second>\d\d(\.\d*)?)?)?(?P<tz>Z|[+-]\d\d:?\d\d)?" | |
).set_name("ISO8601 datetime") | |
"ISO8601 datetime (``yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)``) - trailing seconds, milliseconds, and timezone optional; accepts separating ``'T'`` or ``' '``" | |
uuid = Regex(r"[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}").set_name("UUID") | |
"UUID (``xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx``)" | |
_html_stripper = any_open_tag.suppress() | any_close_tag.suppress() | |
def strip_html_tags(s: str, l: int, tokens: ParseResults): | |
"""Parse action to remove HTML tags from web page HTML source | |
Example:: | |
# strip HTML links from normal text | |
text = '<td>More info at the <a href="https://github.com/pyparsing/pyparsing/wiki">pyparsing</a> wiki page</td>' | |
td, td_end = makeHTMLTags("TD") | |
table_text = td + SkipTo(td_end).setParseAction(pyparsing_common.stripHTMLTags)("body") + td_end | |
print(table_text.parseString(text).body) | |
Prints:: | |
More info at the pyparsing wiki page | |
""" | |
return pyparsing_common._html_stripper.transform_string(tokens[0]) | |
_commasepitem = ( | |
Combine( | |
OneOrMore( | |
~Literal(",") | |
+ ~LineEnd() | |
+ Word(printables, exclude_chars=",") | |
+ Opt(White(" \t") + ~FollowedBy(LineEnd() | ",")) | |
) | |
) | |
.streamline() | |
.set_name("commaItem") | |
) | |
comma_separated_list = delimited_list( | |
Opt(quoted_string.copy() | _commasepitem, default="") | |
).set_name("comma separated list") | |
"""Predefined expression of 1 or more printable words or quoted strings, separated by commas.""" | |
upcase_tokens = staticmethod(token_map(lambda t: t.upper())) | |
"""Parse action to convert tokens to upper case.""" | |
downcase_tokens = staticmethod(token_map(lambda t: t.lower())) | |
"""Parse action to convert tokens to lower case.""" | |
# fmt: off | |
url = Regex( | |
# https://mathiasbynens.be/demo/url-regex | |
# https://gist.github.com/dperini/729294 | |
r"^" + | |
# protocol identifier (optional) | |
# short syntax // still required | |
r"(?:(?:(?P<scheme>https?|ftp):)?\/\/)" + | |
# user:pass BasicAuth (optional) | |
r"(?:(?P<auth>\S+(?::\S*)?)@)?" + | |
r"(?P<host>" + | |
# IP address exclusion | |
# private & local networks | |
r"(?!(?:10|127)(?:\.\d{1,3}){3})" + | |
r"(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})" + | |
r"(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})" + | |
# IP address dotted notation octets | |
# excludes loopback network 0.0.0.0 | |
# excludes reserved space >= 224.0.0.0 | |
# excludes network & broadcast addresses | |
# (first & last IP address of each class) | |
r"(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])" + | |
r"(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}" + | |
r"(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))" + | |
r"|" + | |
# host & domain names, may end with dot | |
# can be replaced by a shortest alternative | |
# (?![-_])(?:[-\w\u00a1-\uffff]{0,63}[^-_]\.)+ | |
r"(?:" + | |
r"(?:" + | |
r"[a-z0-9\u00a1-\uffff]" + | |
r"[a-z0-9\u00a1-\uffff_-]{0,62}" + | |
r")?" + | |
r"[a-z0-9\u00a1-\uffff]\." + | |
r")+" + | |
# TLD identifier name, may end with dot | |
r"(?:[a-z\u00a1-\uffff]{2,}\.?)" + | |
r")" + | |
# port number (optional) | |
r"(:(?P<port>\d{2,5}))?" + | |
# resource path (optional) | |
r"(?P<path>\/[^?# ]*)?" + | |
# query string (optional) | |
r"(\?(?P<query>[^#]*))?" + | |
# fragment (optional) | |
r"(#(?P<fragment>\S*))?" + | |
r"$" | |
).set_name("url") | |
# fmt: on | |
# pre-PEP8 compatibility names | |
convertToInteger = convert_to_integer | |
convertToFloat = convert_to_float | |
convertToDate = convert_to_date | |
convertToDatetime = convert_to_datetime | |
stripHTMLTags = strip_html_tags | |
upcaseTokens = upcase_tokens | |
downcaseTokens = downcase_tokens | |
_builtin_exprs = [ | |
v for v in vars(pyparsing_common).values() if isinstance(v, ParserElement) | |
] | |