Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 26 additions & 7 deletions contentcuration/contentcuration/tests/test_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,29 +12,43 @@ def number_tests():
("-100", -100),
("1,000,000", 1000000),
("-1,000,000", -1000000),
("1,00,00", 1),
("1,00,00", None),
("1.2", 1.2),
("-1.2", -1.2),
("1.-2", 1),
("1.2,00", 1.2),
("1.-2", None),
("1.2,00", None),
("1,000.5", 1000.5),
("-1,000.5", -1000.5),
("1 1/2", 1.5),
("1,000 1/2", 1000.5),
("-1 1/2", -1.5),
("1/2/123", 0.5),
("12/0", 12),
("1/2/123", None),
("12/0", None),
("50%", 0.5),
("-4.5%", -0.045),
("100%", 1),
("1/2%", 0.005),
("1 1/2%", 0.015),
("1.1.23", 1.1),
("1.1.23", None),
("2.3e10", 2.3e10),
("-2.3e10", -2.3e10),
("2.3e-10", 2.3e-10),
("1,000e+-3", 1),
("1,000e+-3", None),
("eeee", None),
("1:3", None),
("5 apples", None),
("x = 4.62", None),
(" 4.62 ", 4.62),
("0.0", 0),
("0%", 0),
("0/5", 0),
("2e0", 2),
(r"4\.62", 4.62),
(r"249\.6", 249.6),
(r"8\.5", 8.5),
(r"0\.0025", 0.0025),
(r"-4\.5%", -0.045),
(r"2\.3e10", 2.3e10),
]


Expand All @@ -54,6 +68,11 @@ def test_numbers(number_tests):
)


def test_non_string_answer_returns_none():
assert extract_value(10) is None
assert extract_value(1.5) is None


def test_jsons(json_tests):
for val1, val2 in json_tests:
assert load_json_string(val1) == val2, "JSONs don't match: {} != {}".format(
Expand Down
42 changes: 23 additions & 19 deletions contentcuration/contentcuration/utils/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
DIGIT: [0-9]
NON_ZERO_DIGIT: [1-9]
SIGN: -{0,1}
EXPONENT: [DECIMAL | INTEGER]e+{0,1}[INTEGER]
EXPONENT: [DECIMAL | INTEGER]e[+|-]{0,1}[FORMATTED_INT | UNFORMATTED_INT]

"""
import json
Expand Down Expand Up @@ -76,7 +76,7 @@
)
PERCENTAGE = re.compile("({num})%".format(num=VALID_NUMBER.pattern))
EXPONENT = re.compile(
"((?:{decimal}|{integer})e\\+?{integer})".format(
"((?:{decimal}|{integer})e(?!\\+-)\\+?{integer})".format(
decimal=DECIMAL.pattern, integer=INTEGER.pattern
)
)
Expand All @@ -88,37 +88,41 @@ def extract_value(text):

def parse_valid_number(text):
try:
return (
parse_exponent(text)
or parse_percentage(text)
or parse_mixed_number(text)
or parse_fraction(text)
or parse_decimal(text)
or parse_integer(text)
)
text = text.strip().replace(r"\.", ".")
for parse in (
parse_exponent,
parse_percentage,
parse_mixed_number,
parse_fraction,
parse_decimal,
parse_integer,
):
value = parse(text)
if value is not None:
return value
except Exception:
return None


def parse_integer(text):
match = INTEGER.search(text)
match = INTEGER.fullmatch(text)
return match and float(to_en(match.group(1)))


def parse_decimal(text):
match = DECIMAL.search(text)
match = DECIMAL.fullmatch(text)
return match and float(to_en(match.group(1)))


def parse_fraction(text):
match = FRACTION.search(text)
match = FRACTION.fullmatch(text)
return match and float(parse_integer(match.group(2))) / float(
parse_integer(match.group(3))
)


def parse_mixed_number(text):
match = MIXED_NUMBER.search(text)
match = MIXED_NUMBER.fullmatch(text)
if match:
number = parse_integer(match.group(1))
return (abs(number) + parse_fraction(match.group(3))) * (
Expand All @@ -128,18 +132,18 @@ def parse_mixed_number(text):


def parse_percentage(text):
match = PERCENTAGE.search(text)
match = PERCENTAGE.fullmatch(text)
return match and extract_value(match.group(1)) / float(100)


def parse_exponent(text):
eval_str = None
match = EXPONENT.search(text)
match = EXPONENT.fullmatch(text)
if match:
val1 = extract_value(match.group(2) or match.group(4))
val2 = extract_value(match.group(5))
eval_str = val1 and val2 and "{int}e{exp}".format(int=val1, exp=int(val2))
return eval_str and eval(to_en(eval_str))
if val1 is not None and val2 is not None:
return eval(to_en("{int}e{exp}".format(int=val1, exp=int(val2))))
return None


def to_en(text):
Expand Down
Loading