From 73ad228ac298c1372e2aa935c908654bc6cd5836 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Sun, 9 Aug 2020 17:15:38 -0700 Subject: [PATCH 01/13] adding a quick deeply nested data sample --- docs/samples/nested_data_1.py | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 docs/samples/nested_data_1.py diff --git a/docs/samples/nested_data_1.py b/docs/samples/nested_data_1.py new file mode 100644 index 00000000..baa3a10e --- /dev/null +++ b/docs/samples/nested_data_1.py @@ -0,0 +1,36 @@ +data = \ +{ + "status": "ok", + "version": "v1", + "schedule_id": "2014-03-18", + "table_name": "Outer table", + "data": [ + { + "data": [ + { + "year": "2009", + "values": [ + { + "version": "v1", + "data": [ + { + "data": [ + { + "year": "2009", + "values": [42], + } + ], + "schedule_id": "2014-03-18", + "table_name": "Inner Table", + "unit": "Percent" + } + ] + } + ], + } + ], + } + ] +} + +print(data) From 5ff0e800bf72ae00d1fc33f904007091522fc73c Mon Sep 17 00:00:00 2001 From: Kurt Rose <=> Date: Sun, 9 Aug 2020 18:35:29 -0700 Subject: [PATCH 02/13] working out match/ref example --- glom/test/demo_test.py | 44 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 glom/test/demo_test.py diff --git a/glom/test/demo_test.py b/glom/test/demo_test.py new file mode 100644 index 00000000..0d343967 --- /dev/null +++ b/glom/test/demo_test.py @@ -0,0 +1,44 @@ +""" +structured demo code as tests to keep us honest in ohnly showing working code +""" +import json + +import pytest + +from glom import Ref, Match, Switch, Auto, T, glom, Or, GlomError, M, And + + + +def test_json_check(): + # we can define a self-recursive spec that + # checks a data structure for JSON compatibility + json_spec = Match(Ref( + 'json', Switch({ + dict: {str: Ref('json')}, + list: [Ref('json')], + Or(int, float, str): T, + }) + )) + glom({'customers': ['alice', 'bob']}, json_spec) + # why might we want to do this? + # first, we get a nice error tree + # we also have very fine grained control over behaviors + # maybe we want to be more strict than the standard library + json.dumps({1: 1}) # {"1": 1} + with pytest.raises(GlomError): + glom({1: 1}, json_spec) + # maybe we want to customize a little bit + # for example, lets say we want to ensure that + # integers are 'float-safe' (can be losslessly + # converted to an IEEE 754 double precision float) + json_spec = Match(Ref( + 'json', Switch({ + dict: {str: Ref('json')}, + list: [Ref('json')], + Or(float, str): T, + And(int, M > - 2 ** 52, M < 2 ** 52): T, + }) + )) + glom(2**51, json_spec) + with pytest.raises(GlomError): + glom(2**52, json_spec) From e2105e26d8ca905d348af4dabc0160c15b44c9bf Mon Sep 17 00:00:00 2001 From: Kurt Rose <=> Date: Sun, 9 Aug 2020 18:49:38 -0700 Subject: [PATCH 03/13] added self-recursive serialization to json dumps example --- glom/test/demo_test.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/glom/test/demo_test.py b/glom/test/demo_test.py index 0d343967..689ee68d 100644 --- a/glom/test/demo_test.py +++ b/glom/test/demo_test.py @@ -42,3 +42,25 @@ def test_json_check(): glom(2**51, json_spec) with pytest.raises(GlomError): glom(2**52, json_spec) + # maybe we want to dump an object but still + # check that its output is valid + class Message(object): + def __init__(self, value): + self.value = value + + def as_dict(self): + return {'type': 'message', 'value': self.value} + + json_spec = Match(Ref( + 'json', Switch({ + dict: {str: Ref('json')}, + list: [Ref('json')], + Or(int, float, str): T, + T.as_dict: Auto((T.as_dict(), Match(Ref('json')))) + }) + )) + + glom(Message(1), json_spec) + glom(Message(Message(1)), json_spec) + with pytest.raises(GlomError): + glom(Message(object), json_spec) From fbe5b4b097ed9f7db5d5581817eb43e5c6937962 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Sun, 9 Aug 2020 22:35:06 -0700 Subject: [PATCH 04/13] take 1.5 --- docs/samples/{nested_data_1.py => intro.py} | 3 +++ 1 file changed, 3 insertions(+) rename docs/samples/{nested_data_1.py => intro.py} (85%) diff --git a/docs/samples/nested_data_1.py b/docs/samples/intro.py similarity index 85% rename from docs/samples/nested_data_1.py rename to docs/samples/intro.py index baa3a10e..a8f26d7e 100644 --- a/docs/samples/nested_data_1.py +++ b/docs/samples/intro.py @@ -34,3 +34,6 @@ } print(data) + +res = data.get('data', [{}])[0].get('data')[0].get('values', [{}])[0].get('data')[0].get('schedule_id') +print(res) From 09e9cea15b3c76d490f2031efdf598b533ae2efe Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Wed, 12 Aug 2020 23:59:55 -0700 Subject: [PATCH 05/13] the transformation --- docs/samples/intro.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/samples/intro.py b/docs/samples/intro.py index a8f26d7e..be095a32 100644 --- a/docs/samples/intro.py +++ b/docs/samples/intro.py @@ -37,3 +37,16 @@ res = data.get('data', [{}])[0].get('data')[0].get('values', [{}])[0].get('data')[0].get('schedule_id') print(res) + + +## The transformation + +from glom import glom + +target = {'data': {'id': 2, 'date': '1999-01-01'}} + +response = {'id': glom(target, 'data.id'), + 'date': glom(target, 'data.date')} + +response = glom(target, {'id': 'data.id', + 'date': 'data.date'}) From cfda1bf5290fd5d29ebd56903bde9f74eae2d243 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Thu, 13 Aug 2020 00:15:20 -0700 Subject: [PATCH 06/13] extra space dangit --- docs/samples/intro.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/samples/intro.py b/docs/samples/intro.py index be095a32..ac1eaeb0 100644 --- a/docs/samples/intro.py +++ b/docs/samples/intro.py @@ -49,4 +49,4 @@ 'date': glom(target, 'data.date')} response = glom(target, {'id': 'data.id', - 'date': 'data.date'}) + 'date': 'data.date'}) From ca35e86633fcb25352a81333aae4005edf9629b6 Mon Sep 17 00:00:00 2001 From: Kurt Rose <=> Date: Thu, 13 Aug 2020 09:16:30 -0700 Subject: [PATCH 07/13] json_spec_r --- glom/test/demo_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/glom/test/demo_test.py b/glom/test/demo_test.py index 689ee68d..81514ae1 100644 --- a/glom/test/demo_test.py +++ b/glom/test/demo_test.py @@ -60,6 +60,21 @@ def as_dict(self): }) )) + # what would the alternative, self-recursive version look like? + def json_spec_r(val): + if type(val) is dict: + for key in val: + assert isinstance(key, str) + return { + key: json_spec_r(sub_val) for key, sub_val in val.items()} + if type(val) is list: + return [json_spec_r(sub_val) for sub_val in val] + if type(val) in (int, float, str): + return val + if hasattr(val, "as_dict"): + return json_spec_r(val.as_dict()) + raise TypeError('no match') + glom(Message(1), json_spec) glom(Message(Message(1)), json_spec) with pytest.raises(GlomError): From ef6790fbdb4fb9f9ad3a69f27bd0aff40bf2fc83 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Thu, 13 Aug 2020 23:11:19 -0700 Subject: [PATCH 08/13] couple tweaks to the intro code --- docs/samples/intro.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/docs/samples/intro.py b/docs/samples/intro.py index ac1eaeb0..ebef5382 100644 --- a/docs/samples/intro.py +++ b/docs/samples/intro.py @@ -43,10 +43,23 @@ from glom import glom -target = {'data': {'id': 2, 'date': '1999-01-01'}} +target = {'ID': 2, 'data': {'isoDate': '1999-01-01'}} -response = {'id': glom(target, 'data.id'), - 'date': glom(target, 'data.date')} +response = {'id': glom(target, 'ID'), + 'date': glom(target, 'data.isoDate')} -response = glom(target, {'id': 'data.id', - 'date': 'data.date'}) +response = glom(target, {'id': 'ID', + 'date': 'data.isoDate'}) +print(response) + + +EMAIL_SPEC = {'id': 'email_id', + 'email': 'email_addr', + 'type': 'email_type'} + +def get_all_emails(filters): + queryset = Email.objects.filter(**filters) + + all_emails_spec = {'results': [EMAIL_SPEC]} + + return glom(queryset, all_emails_spec) From de417f080ec38f2278a25da3a3f614c3792c67ff Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Fri, 14 Aug 2020 00:28:27 -0700 Subject: [PATCH 09/13] add some middle material --- docs/samples/mid_specs.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 docs/samples/mid_specs.py diff --git a/docs/samples/mid_specs.py b/docs/samples/mid_specs.py new file mode 100644 index 00000000..3c91acad --- /dev/null +++ b/docs/samples/mid_specs.py @@ -0,0 +1,26 @@ + +from glom import glom, T + +spec = T['a']['b']['c'] + +target = {'a': {'b': {'c': 'd'}}} + +output = glom(target, spec) +assert output == 'd' + +output = glom(target, T['a']['b']['c'].upper()) +assert output == 'D' + +output = glom(target, T['z'].upper(), default=None) +assert output is None + + + +from glom import glom, Iter + +target = ['1', '2', '1', '3'] + +spec = Iter().map(int).unique().all() + +output = glom(target, spec) +assert output == [1, 2, 3] From f685a8fb5a76fb0ce56d8259182413c0dac0f381 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Fri, 14 Aug 2020 22:37:49 -0700 Subject: [PATCH 10/13] sample code bump --- docs/samples/intro.py | 8 +++--- docs/samples/mid_specs.py | 54 +++++++++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 14 deletions(-) diff --git a/docs/samples/intro.py b/docs/samples/intro.py index ebef5382..7cce92c0 100644 --- a/docs/samples/intro.py +++ b/docs/samples/intro.py @@ -2,7 +2,7 @@ { "status": "ok", "version": "v1", - "schedule_id": "2014-03-18", + "sched_date": "2016-05-12", "table_name": "Outer table", "data": [ { @@ -20,7 +20,7 @@ "values": [42], } ], - "schedule_id": "2014-03-18", + "sched_date": "2014-03-18", "table_name": "Inner Table", "unit": "Percent" } @@ -35,7 +35,9 @@ print(data) -res = data.get('data', [{}])[0].get('data')[0].get('values', [{}])[0].get('data')[0].get('schedule_id') +data.get('data', [{}])[0] + +res = data.get('data', [{}])[0].get('data')[0].get('values', [{}])[0].get('data')[0].get('sched_date') print(res) diff --git a/docs/samples/mid_specs.py b/docs/samples/mid_specs.py index 3c91acad..df020bda 100644 --- a/docs/samples/mid_specs.py +++ b/docs/samples/mid_specs.py @@ -1,20 +1,24 @@ +# assignment -from glom import glom, T +from glom import glom, Assign -spec = T['a']['b']['c'] +target = {'a': [{'b': 'c'}, {'d': None}]} + +# let's set the 'd' key to 'e' +glom(target, Assign('a.1.d', 'e')) + +assert target['a'][1]['d'] == 'e' -target = {'a': {'b': {'c': 'd'}}} -output = glom(target, spec) -assert output == 'd' +from glom import Spec -output = glom(target, T['a']['b']['c'].upper()) -assert output == 'D' +# let's assign one target value to another target value +glom(target, Assign('a.1.d', Spec('a.0.b'))) -output = glom(target, T['z'].upper(), default=None) -assert output is None +assert target['a'] == [{'b': 'c'}, {'d': 'c'}] +# Iter from glom import glom, Iter @@ -22,5 +26,33 @@ spec = Iter().map(int).unique().all() -output = glom(target, spec) -assert output == [1, 2, 3] +result = glom(target, spec) +assert result == [1, 2, 3] + +# T + +from glom import glom, T + +target = {'a': {'b': {'c': 'd'}}} + +spec = T['a']['b']['c'] + +result = glom(target, spec) +assert result == 'd' + +result = glom(target, T['a']['b']['c'].upper()) +assert result == 'D' + +result = glom(target, T['z'].upper(), default=None) +assert result is None + + +class Contact(object): + def __init__(self, name, details_dict): + self.name = name + self.details = details_dict + +frank = Contact('Frank', {'emails': ['FRANK@NothingButHotdogs.BIZ']}) + +result = glom(frank, T.details['emails'][0].lower()) +assert result == 'frank@nothingbuthotdogs.biz' From 3cce459270e182c2c2f1b0e696b7108abcdf1692 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Sat, 15 Aug 2020 00:55:37 -0700 Subject: [PATCH 11/13] the result has always been 'output' (see FAQ) --- docs/samples/mid_specs.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/samples/mid_specs.py b/docs/samples/mid_specs.py index df020bda..be485753 100644 --- a/docs/samples/mid_specs.py +++ b/docs/samples/mid_specs.py @@ -26,8 +26,8 @@ spec = Iter().map(int).unique().all() -result = glom(target, spec) -assert result == [1, 2, 3] +output = glom(target, spec) +assert output == [1, 2, 3] # T @@ -37,14 +37,14 @@ spec = T['a']['b']['c'] -result = glom(target, spec) -assert result == 'd' +output = glom(target, spec) +assert output == 'd' -result = glom(target, T['a']['b']['c'].upper()) -assert result == 'D' +output = glom(target, T['a']['b']['c'].upper()) +assert output == 'D' -result = glom(target, T['z'].upper(), default=None) -assert result is None +output = glom(target, T['z'].upper(), default=None) +assert output is None class Contact(object): @@ -54,5 +54,5 @@ def __init__(self, name, details_dict): frank = Contact('Frank', {'emails': ['FRANK@NothingButHotdogs.BIZ']}) -result = glom(frank, T.details['emails'][0].lower()) -assert result == 'frank@nothingbuthotdogs.biz' +output = glom(frank, T.details['emails'][0].lower()) +assert output == 'frank@nothingbuthotdogs.biz' From d5b1d834dd0b4c1ce218c5137f7f9c4f35695cfa Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Sat, 15 Aug 2020 13:23:36 -0700 Subject: [PATCH 12/13] another glom vs raw python comparison --- docs/samples/intro.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/samples/intro.py b/docs/samples/intro.py index 7cce92c0..052eacaa 100644 --- a/docs/samples/intro.py +++ b/docs/samples/intro.py @@ -55,11 +55,40 @@ print(response) +def get_response(in_data): + ret = {} + + try: + ret['id'] = in_data['ID'] + except (KeyError, TypeError): + pass # TODO + + try: + data = in_data['data'] + except KeyError: + pass # TODO + + try: + ret['date'] = data['isoDate'] + except (KeyError, TypeError): + pass # TODO + + return ret + +assert get_response(target) == response + + EMAIL_SPEC = {'id': 'email_id', 'email': 'email_addr', 'type': 'email_type'} -def get_all_emails(filters): + +def get_email(email_id): + email = Email.objects.get(email_id=email_id) + return glom(email, EMAIL_SPEC) + + +def get_all_emails(**filters): queryset = Email.objects.filter(**filters) all_emails_spec = {'results': [EMAIL_SPEC]} From 19862525f72fcc49b9aadb5c4a605a070b7e2f74 Mon Sep 17 00:00:00 2001 From: Mahmoud Hashemi Date: Sat, 15 Aug 2020 14:33:43 -0700 Subject: [PATCH 13/13] add spec type example --- docs/samples/adv.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 docs/samples/adv.py diff --git a/docs/samples/adv.py b/docs/samples/adv.py new file mode 100644 index 00000000..7cb60fae --- /dev/null +++ b/docs/samples/adv.py @@ -0,0 +1,15 @@ + +from glom import glom + + +class Seq(object): + "Applies a list of specs to a single target, returning a list of results" + def __init__(self, *subspecs): + self.subspecs = subspecs + + def glomit(self, target, scope): + return [scope[glom](target, spec, scope) for spec in self.subspecs] + +output = glom('1', Seq(float, int)) + +assert output == [1.0, 1]