diff --git a/glom/core.py b/glom/core.py index c80ea43f..c6db097e 100644 --- a/glom/core.py +++ b/glom/core.py @@ -55,40 +55,6 @@ _type_type = type _MISSING = make_sentinel('_MISSING') -SKIP = make_sentinel('SKIP') -SKIP.__doc__ = """ -The ``SKIP`` singleton can be returned from a function or included -via a :class:`~glom.Val` to cancel assignment into the output -object. - ->>> target = {'a': 'b'} ->>> spec = {'a': lambda t: t['a'] if t['a'] == 'a' else SKIP} ->>> glom(target, spec) -{} ->>> target = {'a': 'a'} ->>> glom(target, spec) -{'a': 'a'} - -Mostly used to drop keys from dicts (as above) or filter objects from -lists. - -.. note:: - - SKIP was known as OMIT in versions 18.3.1 and prior. Versions 19+ - will remove the OMIT alias entirely. -""" -OMIT = SKIP # backwards compat, remove in 19+ - -STOP = make_sentinel('STOP') -STOP.__doc__ = """ -The ``STOP`` singleton can be used to halt iteration of a list or -execution of a tuple of subspecs. - ->>> target = range(10) ->>> spec = [lambda x: x if x < 5 else STOP] ->>> glom(target, spec) -[0, 1, 2, 3, 4] -""" LAST_CHILD_SCOPE = make_sentinel('LAST_CHILD_SCOPE') LAST_CHILD_SCOPE.__doc__ = """ @@ -121,6 +87,65 @@ _PKG_DIR_PATH = os.path.dirname(os.path.abspath(__file__)) + +# these can't be sub-classes of GlomError b/c they are a different +# type of flow control mechanism; we don't want Or() to change +# its behavior b/c of skip or stop +class GlomSkip(Exception): pass + +class GlomStop(Exception): pass + + +class _SkipType(object): + """ + The ``SKIP`` singleton can be returned from a function or included + via a :class:`~glom.Val` to cancel assignment into the output + object. + + >>> target = {'a': 'b'} + >>> spec = {'a': lambda t: t['a'] if t['a'] == 'a' else SKIP} + >>> glom(target, spec) + {} + >>> target = {'a': 'a'} + >>> glom(target, spec) + {'a': 'a'} + + Mostly used to drop keys from dicts (as above) or filter objects from + lists. + + .. note:: + + SKIP was known as OMIT in versions 18.3.1 and prior. Versions 19+ + will remove the OMIT alias entirely. + """ + def glomit(self, target, scope): + raise GlomSkip() + + def __repr__(self): + return "SKIP" + + +class _StopType(object): + """ + The ``STOP`` singleton can be used to halt iteration of a list or + execution of a tuple of subspecs. + + >>> target = range(10) + >>> spec = [lambda x: x if x < 5 else STOP] + >>> glom(target, spec) + [0, 1, 2, 3, 4] + """ + def glomit(self, target, scope): + raise GlomStop() + + def __repr__(self): + return "STOP" + +SKIP = _SkipType() +OMIT = SKIP # backwards compat, remove in 19+ + +STOP = _StopType() + class GlomError(Exception): """The base exception for all the errors that might be raised from :func:`glom` processing logic. @@ -1730,9 +1755,12 @@ def _get_sequence_item(target, index): def _handle_dict(target, spec, scope): ret = type(spec)() # TODO: works for dict + ordereddict, but sufficient for all? for field, subspec in spec.items(): - val = scope[glom](target, subspec, scope) - if val is SKIP: + try: + val = scope[glom](target, subspec, scope) + except GlomSkip: continue + except GlomStop: + break if type(field) in (Spec, TType): field = scope[glom](target, field, scope) ret[field] = val @@ -1751,10 +1779,11 @@ def _handle_list(target, spec, scope): base_path = scope[Path] for i, t in enumerate(iterator): scope[Path] = base_path + [i] - val = scope[glom](t, subspec, scope) - if val is SKIP: + try: + val = scope[glom](t, subspec, scope) + except GlomSkip: continue - if val is STOP: + except GlomStop: break ret.append(val) return ret @@ -1764,10 +1793,11 @@ def _handle_tuple(target, spec, scope): res = target for subspec in spec: scope = chain_child(scope) - nxt = scope[glom](res, subspec, scope) - if nxt is SKIP: + try: + nxt = scope[glom](res, subspec, scope) + except GlomSkip: continue - if nxt is STOP: + except GlomStop: break res = nxt if not isinstance(subspec, list): diff --git a/glom/grouping.py b/glom/grouping.py index cae8fca9..65c161b7 100644 --- a/glom/grouping.py +++ b/glom/grouping.py @@ -7,7 +7,7 @@ from boltons.typeutils import make_sentinel -from .core import glom, MODE, SKIP, STOP, TargetRegistry, Path, T, BadSpec, _MISSING +from .core import glom, MODE, GlomSkip, GlomStop, TargetRegistry, Path, T, BadSpec, _MISSING ACC_TREE = make_sentinel('ACC_TREE') @@ -30,6 +30,12 @@ """ +DONE = make_sentinel('DONE') +DONE.__doc__ = """ +internal marker used to keep track that a branch has finished processing +""" + + def target_iter(target, scope): iterate = scope[TargetRegistry].get_handler('iterate', target, path=scope[Path]) @@ -86,9 +92,10 @@ def glomit(self, target, scope): ret = None for t in target_iter(target, scope): - last, ret = ret, scope[glom](t, self.spec, scope) - if ret is STOP: - return last + try: + ret = scope[glom](t, self.spec, scope) + except GlomStop: + break return ret def __repr__(self): @@ -118,28 +125,32 @@ def GROUP(target, spec, scope): if _spec_type is dict: done = True for keyspec, valspec in spec.items(): - if tree.get(keyspec, None) is STOP: + if tree.get(keyspec, None) is DONE: continue - key = recurse(keyspec) - if key is SKIP: - done = False # SKIP means we still want more vals + try: + key = recurse(keyspec) + except GlomSkip: + done = False # skip means we still want more vals continue - if key is STOP: - tree[keyspec] = STOP + except GlomStop: + tree[keyspec] = DONE continue if key not in acc: # TODO: guard against key == id(spec) tree[key] = {} scope[ACC_TREE] = tree[key] - result = recurse(valspec) - if result is STOP: - tree[keyspec] = STOP + try: + result = recurse(valspec) + except GlomStop: + tree[keyspec] = DONE continue - done = False # SKIP or returning a value means we still want more vals - if result is not SKIP: + except GlomSkip: + pass + else: acc[key] = result + done = False # skip or returning a value means we still want more vals if done: - return STOP + raise GlomStop() return acc elif _spec_type is list: for valspec in spec: @@ -147,10 +158,11 @@ def GROUP(target, spec, scope): # doesn't make sense due to arity mismatch. did you mean [Auto({...})] ? raise BadSpec('dicts within lists are not' ' allowed while in Group mode: %r' % spec) - result = recurse(valspec) - if result is STOP: - return STOP - if result is not SKIP: + try: + result = recurse(valspec) + except GlomSkip: # let GlomStop bubble up + pass + else: acc.append(result) return acc raise ValueError("{} not a valid spec type for Group mode".format(_spec_type)) # pragma: no cover @@ -167,9 +179,9 @@ class First(object): def agg(self, target, tree): if self not in tree: - tree[self] = STOP + tree[self] = DONE return target - return STOP + raise GlomStop() def __repr__(self): return '%s()' % self.__class__.__name__ @@ -310,7 +322,7 @@ def glomit(self, target, scope): scope[ACC_TREE] = tree[self][1] tree[self][0] += 1 if tree[self][0] > self.n: - return STOP + raise GlomStop() return scope[glom](target, self.subspec, scope) def __repr__(self): diff --git a/glom/streaming.py b/glom/streaming.py index 9b83abf7..56cb2b5a 100644 --- a/glom/streaming.py +++ b/glom/streaming.py @@ -22,6 +22,7 @@ from boltons.funcutils import FunctionBuilder from .core import glom, T, STOP, SKIP, _MISSING, Path, TargetRegistry, Call, Spec, S, bbrepr, format_invocation +from .core import GlomSkip, GlomStop from .matching import Check class Iter(object): @@ -101,14 +102,14 @@ def _iterate(self, target, scope): base_path = scope[Path] for i, t in enumerate(iterator): scope[Path] = base_path + [i] - yld = (t if self.subspec is T else scope[glom](t, self.subspec, scope)) - if yld is SKIP: + try: + yld = (t if self.subspec is T else scope[glom](t, self.subspec, scope)) + except GlomSkip: continue - elif yld is self.sentinel or yld is STOP: - # NB: sentinel defaults to STOP so I was torn whether - # to also check for STOP, and landed on the side of - # never letting STOP through. - return + except GlomStop: + break + if yld is self.sentinel: + break yield yld return diff --git a/glom/test/test_basic.py b/glom/test/test_basic.py index 2bfb5038..06e460b9 100644 --- a/glom/test/test_basic.py +++ b/glom/test/test_basic.py @@ -5,7 +5,7 @@ import pytest from glom import glom, SKIP, STOP, Path, Inspect, Coalesce, CoalesceError, Val, Call, T, S, Invoke, Spec, Ref -from glom import Auto, Fill, Iter, A, Vars, Val, Literal, GlomError +from glom import Auto, Fill, Iter, A, Vars, Val, Literal, GlomError, Match, And, Or, M import glom.core as glom_core from glom.core import UP, ROOT, bbformat, bbrepr @@ -112,7 +112,7 @@ def test_skip(): 'n': 'o'} res = glom(target, {'a': 'a.b', - 'z': Coalesce('x', 'y', default=SKIP)}) + 'z': Coalesce('x', 'y', SKIP)}) assert res['a'] == 'c' # sanity check assert 'x' not in target @@ -121,25 +121,26 @@ def test_skip(): # test that skip works on lists target = range(7) - res = glom(target, [lambda t: t if t % 2 else SKIP]) + # TODO: rewrite this when T supports % -- (M(T % 2) == 1) | SKIP + res = glom(target, [Match(Or(lambda t: t % 2 == 1, SKIP))]) assert res == [1, 3, 5] # test that skip works on chains (enable conditional applications of transforms) target = range(7) # double each value if it's even, but convert all values to floats - res = glom(target, [(lambda x: x * 2 if x % 2 == 0 else SKIP, float)]) + # TODO: rewrite this when T supports % and *: M(T % 2 == 0) & (T * 2) | SKIP + res = glom(target, [(Or(And(Match(lambda t: t % 2 == 0), lambda x: x * 2), SKIP), float)]) assert res == [0.0, 1.0, 4.0, 3.0, 8.0, 5.0, 12.0] def test_stop(): # test that stop works on iterables - target = iter([0, 1, 2, STOP, 3, 4]) - assert glom(target, [T]) == [0, 1, 2] + assert glom([0, 1, 2, 3, 4], [(M < 3) | STOP]) == [0, 1, 2] # test that stop works on chains (but doesn't stop iteration up the stack) target = ['a', ' b', ' c ', ' ', ' done'] assert glom(target, [(lambda x: x.strip(), - lambda x: x if x else STOP, + M | STOP, lambda x: x[0])]) == ['a', 'b', 'c', '', 'd'] return diff --git a/glom/test/test_check.py b/glom/test/test_check.py index f498442f..a3a14894 100644 --- a/glom/test/test_check.py +++ b/glom/test/test_check.py @@ -1,7 +1,7 @@ from pytest import raises -from glom import glom, Check, CheckError, Coalesce, SKIP, STOP, T +from glom import glom, Check, CheckError, Coalesce, SKIP, STOP, T, Or try: unicode @@ -10,19 +10,17 @@ def test_check_basic(): - assert glom([0, SKIP], [T]) == [0] # sanity check SKIP - target = [{'id': 0}, {'id': 1}, {'id': 2}] # check that skipping non-passing values works - assert glom(target, ([Coalesce(Check('id', equal_to=0), default=SKIP)], T[0])) == {'id': 0} - assert glom(target, ([Check('id', equal_to=0, default=SKIP)], T[0])) == {'id': 0} + assert glom(target, ([Coalesce(Check('id', equal_to=0), SKIP)], T[0])) == {'id': 0} + assert glom(target, ([Or(Check('id', equal_to=0), SKIP)], T[0])) == {'id': 0} # check that stopping iteration on non-passing values works - assert glom(target, [Check('id', equal_to=0, default=STOP)]) == [{'id': 0}] + assert glom(target, [Or(Check('id', equal_to=0), STOP)]) == [{'id': 0}] # check that stopping chain execution on non-passing values works - spec = (Check(validate=lambda x: len(x) > 0, default=STOP), T[0]) + spec = (Or(Check(validate=lambda x: len(x) > 0), STOP), T[0]) assert glom('hello', spec) == 'h' assert glom('', spec) == '' # would fail with IndexError if STOP didn't work @@ -33,9 +31,9 @@ def test_check_basic(): assert repr(Check(T(len), validate=sum)) == 'Check(T(len), validate=sum)' target = [1, u'a'] - assert glom(target, [Check(type=unicode, default=SKIP)]) == ['a'] + assert glom(target, [Or(Check(type=unicode), SKIP)]) == ['a'] assert glom(target, [Check(type=(unicode, int))]) == [1, 'a'] - assert glom(target, [Check(instance_of=unicode, default=SKIP)]) == ['a'] + assert glom(target, [Or(Check(instance_of=unicode), SKIP)]) == ['a'] assert glom(target, [Check(instance_of=(unicode, int))]) == [1, 'a'] target = ['1'] diff --git a/glom/test/test_grouping.py b/glom/test/test_grouping.py index 0f7aa0ac..312223cb 100644 --- a/glom/test/test_grouping.py +++ b/glom/test/test_grouping.py @@ -2,7 +2,7 @@ from pytest import raises -from glom import glom, T, SKIP, STOP, Auto, BadSpec +from glom import glom, T, SKIP, STOP, Auto, BadSpec, M, Or, Match from glom.grouping import Group, First, Avg, Max, Min, Sample, Limit from glom.reduction import Merge, Flatten, Sum, Count @@ -19,19 +19,19 @@ def test_corner_cases(): target = range(5) # immediate stop dict - assert glom(target, Group({(lambda t: STOP): [T]})) == {} + assert glom(target, Group({STOP: [T]})) == {} # immediate stop list - assert glom(target, Group([lambda t: STOP])) == [] + assert glom(target, Group([STOP])) == [] # dict key SKIP - assert glom(target, Group({(lambda t: SKIP if t < 3 else t): T})) == {3: 3, 4: 4} + assert glom(target, Group({(M > 2) | SKIP: T})) == {3: 3, 4: 4} # dict val SKIP - assert glom(target, Group({T: lambda t: t if t % 2 else SKIP})) == {3: 3, 1: 1} + assert glom(target, Group({T: Or(Match(lambda t: t % 2), SKIP)})) == {3: 3, 1: 1} # list val SKIP - assert glom(target, Group([lambda t: t if t % 2 else SKIP])) == [1, 3] + assert glom(target, Group([Or(Match(lambda t: t % 2), SKIP)])) == [1, 3] # embedded auto spec (lol @ 0 being 0 bit length) assert glom(target, Group({Auto(('bit_length', T())): [T]})) == {0: [0], 1: [1], 2: [2, 3], 3: [4]} diff --git a/glom/test/test_match.py b/glom/test/test_match.py index 31767776..5b039858 100644 --- a/glom/test/test_match.py +++ b/glom/test/test_match.py @@ -217,7 +217,7 @@ def test_pattern_matching(): def test_examples(): assert glom(8, (M > 7) & Val(7)) == 7 assert glom(range(10), [(M > 7) & Val(7) | T]) == [0, 1, 2, 3, 4, 5, 6, 7, 7, 7] - assert glom(range(10), [(M > 7) & Val(SKIP) | T]) == [0, 1, 2, 3, 4, 5, 6, 7] + assert glom(range(10), [(M > 7) & SKIP | T]) == [0, 1, 2, 3, 4, 5, 6, 7] def test_reprs(): @@ -369,7 +369,7 @@ def nullable_list_of(*items): def test_clamp(): assert glom(range(10), [(M < 7) | Val(7)]) == [0, 1, 2, 3, 4, 5, 6, 7, 7, 7] - assert glom(range(10), [(M < 7) | Val(SKIP)]) == [0, 1, 2, 3, 4, 5, 6] + assert glom(range(10), [(M < 7) | SKIP]) == [0, 1, 2, 3, 4, 5, 6] def test_json_ref(): @@ -433,20 +433,20 @@ def test_check_ported_tests(): target = [{'id': 0}, {'id': 1}, {'id': 2}] # check that skipping non-passing values works - assert glom(target, [Coalesce(M(T['id']) == 0, default=SKIP)]) == [{'id': 0}] + assert glom(target, [Coalesce(M(T['id']) == 0, SKIP)]) == [{'id': 0}] # TODO: should M(subspec, default='') work? I lean no. # NB: this is not a very idiomatic use of Match, just brought over for Check reasons - assert glom(target, [Match({'id': And(int, M == 1)}, default=SKIP)]) == [{'id': 1}] - assert glom(target, [Match({'id': And(int, M <= 1)}, default=STOP)]) == [{'id': 0}, {'id': 1}] + assert glom(target, [Match(Or({'id': And(int, M == 1)}, SKIP))]) == [{'id': 1}] + assert glom(target, [Match(Or({'id': And(int, M <= 1)}, STOP))]) == [{'id': 0}, {'id': 1}] # check that stopping chain execution on non-passing values works - spec = (Or(Match(len), Val(STOP)), T[0]) + spec = (Or(Match(len), STOP), T[0]) assert glom('hello', spec, glom_debug=True) == 'h' assert glom('', spec) == '' # would fail with IndexError if STOP didn't work target = [1, u'a'] - assert glom(target, [Match(unicode, default=SKIP)]) == ['a'] + assert glom(target, [Match(Or(unicode, SKIP))]) == ['a'] assert glom(target, Match([Or(unicode, int)])) == [1, 'a'] target = ['1'] diff --git a/glom/test/test_streaming.py b/glom/test/test_streaming.py index 18f94c5f..21089809 100644 --- a/glom/test/test_streaming.py +++ b/glom/test/test_streaming.py @@ -4,7 +4,7 @@ from itertools import count, dropwhile, chain from glom import Iter -from glom import glom, SKIP, STOP, T, Call, Spec, Glommer, Check, SKIP +from glom import glom, SKIP, STOP, T, Call, Spec, Glommer, Check, SKIP, M RANGE_5 = list(range(5)) @@ -19,9 +19,8 @@ def test_iter(): assert list(glom(['1', '2', '3'], (Iter(int), enumerate))) == [(0, 1), (1, 2), (2, 3)] - assert list(glom([1, SKIP, 2], Iter())) == [1, 2] - assert list(glom([1, STOP, 2], Iter())) == [1] - + assert list(glom([1, 2, 3], Iter((M != 2) | SKIP))) == [1, 3] + assert list(glom([1, 2, 3], Iter((M != 2) | STOP))) == [1] with pytest.raises(TypeError): Iter(nonexistent_kwarg=True)