Skip to content

New create_material_variables=True flag #471

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
crunchfu wants to merge 3 commits into
base: master
Choose a base branch
from
Open

New create_material_variables=True flag #471

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3a58aee
New create_material_variables=True flag
aminusfu Oct 16, 2024
d523f35
oops. forgot to update tests
aminusfu Oct 16, 2024
d1b023b
More test fixes. Simpler _var_create_reload_return
aminusfu Oct 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
134 changes: 64 additions & 70 deletions scrunch/datasets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -696,6 +696,17 @@ class DatasetVariablesMixin(Mapping):
Handles dataset variable iteration in a dict-like way
"""

create_material_variables = True
"""New variables created by methods of this class are usually defined by
an expression which is evaluated on the server. If create_material_variables
is True (the default), such an expression is evaluated eagerly, and its
output at that moment is stored as a "material" variable: part of the schema.
Changes to any of the input variables will not update the material variable.
If False, the expression itself is stored instead and re-evaluated on each
read; the resulting "derived" variable is an artifact rather than part of
the schema, and will change if any of its inputs change.
"""

def __getitem__(self, item):
"""
Returns a Variable() instance, `item` can be either a variable alias,
Expand Down Expand Up @@ -744,11 +755,13 @@ def order(self):
self._order = DatasetVariablesOrder(self._catalog, order)
return self._order

def _var_create_reload_return(self, payload):
def _var_create_reload_return(self, entity_body):
"""
helper function for POSTing to variables, reload
the catalog of variables and return newly created var
"""
entity_body.setdefault("derived", not self.create_material_variables)
payload = shoji_entity_wrapper(entity_body)
new_var = self.resource.variables.create(payload)
# needed to update the variables collection
self._reload_variables()
Expand Down Expand Up @@ -1166,13 +1179,13 @@ def create_fill_values(self, variables, name, alias, description=''):
{"map": fill_map}
]
}
payload = shoji_entity_wrapper({
body = {
"alias": alias,
"name": name,
"description": description,
"derivation": fill_expr
})
return self._var_create_reload_return(payload)
}
return self._var_create_reload_return(body)

def create_single_response(self, categories, name, alias, description='',
missing=True, notes=''):
Expand Down Expand Up @@ -1223,13 +1236,13 @@ def create_single_response(self, categories, name, alias, description='',

expr = dict(function='case', args=args + more_args)

payload = shoji_entity_wrapper(dict(
body = dict(
alias=alias,
name=name,
expr=expr,
derivation=expr,
description=description,
notes=notes))
return self._var_create_reload_return(payload)
notes=notes)
return self._var_create_reload_return(body)

def rollup(self, variable_alias, name, alias, resolution, description='',
notes=''):
Expand All @@ -1249,27 +1262,17 @@ def rollup(self, variable_alias, name, alias, resolution, description='',
expr = {
'function': 'rollup',
'args': [
{
'variable': self[variable_alias].url
},
{
'value': resolution
}
{'variable': self[variable_alias].url},
{'value': resolution}
]
}

payload = shoji_entity_wrapper(dict(
body = dict(
alias=alias,
name=name,
expr=expr,
derivation=expr,
description=description,
notes=notes))

new_var = self.resource.variables.create(payload)
# needed to update the variables collection
self._reload_variables()
# return the variable instance
return self[new_var['body']['alias']]
notes=notes)
return self._var_create_reload_return(body)

def derive_multiple_response(self, categories, subvariables, name, alias,
description='', notes='', uniform_basis=False):
Expand Down Expand Up @@ -1320,7 +1323,7 @@ def derive_multiple_response(self, categories, subvariables, name, alias,
categories=categories
)

payload = shoji_entity_wrapper({
body = {
'name': name,
'alias': alias,
'description': description,
Expand All @@ -1336,8 +1339,8 @@ def derive_multiple_response(self, categories, subvariables, name, alias,
]
}]
}
})
return self._var_create_reload_return(payload)
}
return self._var_create_reload_return(body)

def create_multiple_response(self, responses, name, alias, description='',
notes=''):
Expand All @@ -1359,7 +1362,7 @@ def create_multiple_response(self, responses, name, alias, description='',
alias='%s_%d' % (alias, resp['id'])
)

payload = shoji_entity_wrapper({
body = {
'name': name,
'alias': alias,
'description': description,
Expand All @@ -1374,8 +1377,8 @@ def create_multiple_response(self, responses, name, alias, description='',
]
}]
}
})
return self._var_create_reload_return(payload)
}
return self._var_create_reload_return(body)

def variable_aliases(self, include_subvariables=False):
existing_aliases = set()
Expand Down Expand Up @@ -1445,14 +1448,14 @@ def bind_categorical_array(self, name, alias, subvariables, description='',
"subreferences": subreferences
}
}
payload = shoji_entity_wrapper({
body = {
'name': name,
'alias': alias,
'description': description,
'notes': notes,
'derivation': expression
})
return self._var_create_reload_return(payload)
}
return self._var_create_reload_return(body)

def create_numeric(self, alias, name, derivation, description='', notes=''):
"""
Expand All @@ -1464,14 +1467,14 @@ def create_numeric(self, alias, name, derivation, description='', notes=''):
if not hasattr(self.resource, 'variables'):
self.resource.refresh()

payload = shoji_entity_wrapper(dict(
body = dict(
alias=alias,
name=name,
derivation=expr,
description=description,
notes=notes
))
return self._var_create_reload_return(payload)
)
return self._var_create_reload_return(body)

def create_categorical(self, categories, alias, name, multiple, description='',
notes='', missing_case=None, uniform_basis=False):
Expand Down Expand Up @@ -1690,33 +1693,33 @@ def create_variable(self, var_type, name, alias=None, description='',
values = [1,4,5,2,1,3,1]
"""
self._validate_vartypes(var_type, resolution, subvariables, categories)
payload = {
body = {
'type': var_type,
'name': name,
'description': description,
}
if alias:
payload['alias'] = alias
body['alias'] = alias
if resolution:
payload['resolution'] = resolution
body['resolution'] = resolution
if var_type == 'multiple_response' and categories is None:
payload['categories'] = [
body['categories'] = [
{'name': 'Not selected', 'id': NOT_SELECTED_ID, 'numeric_value': 2, 'missing': False},
{'name': 'Selected', 'id': SELECTED_ID, 'numeric_value': 1, 'missing': False, 'selected': True},
]
if categories:
payload['categories'] = categories
body['categories'] = categories
if subvariables:
payload['subreferences'] = []
body['subreferences'] = []
for item in subvariables:
subrefs = {'name': item['name']}
if item.get('alias'):
subrefs['alias'] = item['alias']
payload['subreferences'].append(subrefs)
body['subreferences'].append(subrefs)
if values:
payload['values'] = values
body['values'] = values

self._var_create_reload_return(shoji_entity_wrapper(payload))
self._var_create_reload_return(body)

def copy_variable(self, variable, name, alias, derived=None, subvariable_codes=None):
"""
Expand Down Expand Up @@ -1760,17 +1763,14 @@ def subrefs(_variable, _alias):
derivation = abs_url(variable.resource.body['derivation'],
variable.resource.self)
derivation.pop('references', None)
payload = shoji_entity_wrapper({
'name': name,
'alias': alias,
'derivation': derivation})
body = {'name': name, 'alias': alias, 'derivation': derivation}

if variable.type == _MR_TYPE:
# We are re-executing a multiple_response derivation.
# We need to update the complex `array` function expression
# to contain the new suffixed aliases. Given that the map is
# unordered, we have to iterated and find a name match.
_ob = payload['body']['derivation']['args'][0]['args'][0]
_ob = body['derivation']['args'][0]['args'][0]
subvars = _ob['map']
subreferences = subrefs(variable, alias)
for subref in subreferences:
Expand All @@ -1782,15 +1782,9 @@ def subrefs(_variable, _alias):
else:
derivation = {
'function': 'copy_variable',
'args': [{
'variable': variable_resource.self
}]
'args': [{'variable': variable_resource.self}]
}
payload = shoji_entity_wrapper({
'name': name,
'alias': alias,
'derivation': derivation
})
body = {'name': name, 'alias': alias, 'derivation': derivation}

if "subvariables" in variable_resource.body:
api_subreferences = variable_resource.body["subreferences"]
Expand All @@ -1814,9 +1808,9 @@ def subrefs(_variable, _alias):
derivation["references"] = {"subreferences": subreferences}

if derived is False or derived:
payload['body']['derived'] = derived
body['derived'] = derived

return self._var_create_reload_return(payload)
return self._var_create_reload_return(body)

def combine_categories(self, variable, map, categories, missing=None,
default=None, name='', alias='', description=''):
Expand Down Expand Up @@ -1857,14 +1851,14 @@ def combine_categorical(self, variable, map, categories=None, missing=None,
# TODO: Implement `default` parameter in Crunch API
combinations = combinations_from_map(
map, categories or {}, missing or [])
payload = shoji_entity_wrapper({
body = {
'name': name,
'alias': alias,
'description': description,
'derivation': combine_categories_expr(
variable.resource.self, combinations)
})
return self._var_create_reload_return(payload)
}
return self._var_create_reload_return(body)

def combine_multiple_response(self, variable, map, categories=None, default=None,
name='', alias='', description=''):
Expand All @@ -1889,14 +1883,14 @@ def combine_multiple_response(self, variable, map, categories=None, default=None
# TODO: Implement `default` parameter in Crunch API
responses = responses_from_map(variable, map, categories or {}, alias,
parent_alias)
payload = shoji_entity_wrapper({
body = {
'name': name,
'alias': alias,
'description': description,
'derivation': combine_responses_expr(
variable.resource.self, responses)
})
return self._var_create_reload_return(payload)
}
return self._var_create_reload_return(body)

def cast_summary(self, variable, cast_type):
"""
Expand Down Expand Up @@ -2778,16 +2772,16 @@ def derive_weight(self, targets, alias, name, description=''):
'targets': list(map(list, val.items()))
})

payload = shoji_entity_wrapper({
body = {
'name': name,
'alias': alias,
'description': description,
'derivation': {
'function': 'rake',
'args': _targets
}
})
return self._var_create_reload_return(payload)
}
return self._var_create_reload_return(body)

@property
def is_view(self):
Expand Down Expand Up @@ -3041,7 +3035,7 @@ def edit_categorical(self, categories, rules):
more_args = process_expr(more_args, self.dataset)
# epression value building
expr = dict(function='case', args=args + more_args)
payload = shoji_entity_wrapper(dict(expr=expr))
payload = shoji_entity_wrapper(dict(derivation=expr))
# patch the variable with the new payload
resp = self.resource.patch(payload)
self._reload_variables()
Expand Down
Loading