diff --git a/scrunch/datasets.py b/scrunch/datasets.py index b010f047..71d6d544 100644 --- a/scrunch/datasets.py +++ b/scrunch/datasets.py @@ -1129,7 +1129,7 @@ def create_fill_values(self, variables, name, alias, description=''): if "variable" in else_case and "name" in else_case: raise ValueError("Else case can be either variable or category not both") - aliases = {c["variable"] for c in variables} + aliases = {c["var"] for c in variables} vars_by_alias = self.resource.variables.by("alias") types = {vars_by_alias[al]["type"] for al in aliases} if types != {"categorical"}: @@ -1162,13 +1162,13 @@ def create_fill_values(self, variables, name, alias, description=''): }) expr = {"function": "case", "args": args} - fill_map = {str(cid): {"variable": vars_by_alias[v["variable"]]["id"]} + fill_map = {str(cid): {"var": v["var"]} for cid, v in zip(cat_ids, variables)} - if "variable" in else_case: + if "var" in else_case: # We are in the case of a default fill, replace the -1 with the new # variable - fill_map["-1"] = {"variable": vars_by_alias[else_case["variable"]]["id"]} + fill_map["-1"] = {"var": else_case["var"]} fill_expr = { "function": "fill", @@ -1873,7 +1873,7 @@ def combine_categorical(self, variable, map, categories=None, missing=None, 'alias': alias, 'description': description, 'derivation': combine_categories_expr( - variable.resource.self, combinations) + variable.alias, combinations) }) return self._var_create_reload_return(payload) diff --git a/scrunch/expressions.py b/scrunch/expressions.py index a282bef9..ca713c11 100644 --- a/scrunch/expressions.py +++ b/scrunch/expressions.py @@ -51,7 +51,7 @@ import sys -PY311 = sys.version_info[:2] == (3, 11) +GT_PY_311 = sys.version_info[:2] >= (3, 11) if six.PY2: from urllib import urlencode @@ -169,22 +169,16 @@ def r(lower, upper): return list(range(lower, upper + 1)) -def parse_expr(expr, platonic=False): +def parse_expr(expr): """ Converts a text python-like expression into ZCL tree. - If `platonic` is True, the aliases will use `{"var": }` terms. - :param expr: String with a python-like expression - :param platonic: Boolean, when True variables will be alias `var` terms :return: Dictionary with a ZCL expression """ def _var_term(_var_id): - if platonic: - return {"var": _var_id} - else: - return {'variable': _var_id} + return {"var": _var_id} def _parse(node, parent=None): obj = {} @@ -298,19 +292,14 @@ def _parse(node, parent=None): # We will take the subvariable alias bit from the subscript # and return an object with the array and subvariable alias array_alias = dict(ast.iter_fields(fields[0][1]))["id"] - if PY311: + if GT_PY_311: name_node = dict(ast.iter_fields(fields[1][1])) subvariable_alias = name_node["id"] else: name_node = dict(ast.iter_fields(fields[1][1]))["value"] subscript_fields = dict(ast.iter_fields(name_node)) subvariable_alias = subscript_fields["id"] - if platonic: - return {"var": array_alias, "axes": [subvariable_alias]} - else: - # For non-platonic expressions, keep track of both the array - # and subvariable to make a proper url lookup. - return {"variable": {"array": array_alias, "subvariable": subvariable_alias}} + return {"var": array_alias, "axes": [subvariable_alias]} # "Non-terminal" nodes. else: for _name, _val in fields: @@ -505,17 +494,17 @@ def get_subvariables_resource(var_url, var_index): return {sv['alias'].strip('#'): sv['id'] for sv in sub_variables.values()} -def _get_categories_from_var_index(var_index, var_url): - return var_index[var_url].entity.body.categories +def _get_categories_from_var_index(vars_by_alias, var_alias): + return vars_by_alias[var_alias].entity.body.categories -def adapt_multiple_response(var_url, values, var_index): +def adapt_multiple_response(var_alias, values, vars_by_alias): """ Converts multiple response arguments to column. :return: the new args for multiple_response """ - aliases = get_subvariables_resource(var_url, var_index) + aliases = get_subvariables_resource(var_alias, vars_by_alias) result = [] if all(isinstance(value, int) for value in values): @@ -526,35 +515,36 @@ def adapt_multiple_response(var_url, values, var_index): # scenario var.any([subvar1, subvar2]) # in this scenario, we only want category ids that refers to `selected` categories column = [ - cat.get("id") for cat in _get_categories_from_var_index(var_index, var_url) if cat.get("selected") + cat.get("id") for cat in _get_categories_from_var_index(vars_by_alias, var_alias) if cat.get("selected") ] - variables = [var_id for alias, var_id in aliases.items() if alias in values] + subvars = [sva for sva in aliases if sva in values] - for variable_id in variables: - variable_url = "{}subvariables/{}/".format(var_url, variable_id) + for sva in subvars: result.append({ - "variable": variable_url, + "var": var_alias, + 'axes': [sva], "column": column }) return result, True -def _update_values_for_multiple_response(new_values, values, subitem, var_index, arrays): +def _update_values_for_multiple_response(new_values, values, subitem, vars_by_alias, arrays): """ - Multiple response does not need the `value` key, but it relies on the `column` key - Remove from `arrays` (subvariable list) the ones that should not be considered """ - var_url = subitem.get("variable", "").split("subvariables")[0] + var_alias = subitem.get("var") column = new_values[0].get("column") value = values[0].get("value") - if var_url and var_index[var_url]['type'] == 'multiple_response': + if var_alias and vars_by_alias[var_alias]['type'] == 'multiple_response': if column: values[0]['column'] = column elif value is not None: values[0]['column'] = value values[0].pop("value", None) - arrays[0] = [new_value["variable"] for new_value in new_values] + subvar_ids_by_aliases = {v['alias']: k for k, v in vars_by_alias[var_alias]['entity']['subvariables']['index'].items()} + arrays[0] = [subvar_ids_by_aliases[new_value["axes"][0]] for new_value in new_values] def process_expr(obj, ds): @@ -578,7 +568,7 @@ def ensure_category_ids(subitems, values, arrays, variables=variables): def variable_id(variable_url): return variable_url.split('/')[-2] - def category_ids(var_id, var_value, variables=variables): + def category_ids(var_alias, var_value, variables=variables): value = None if isinstance(var_value, list) or isinstance(var_value, tuple): # {'values': [val1, val2, ...]} @@ -589,7 +579,7 @@ def category_ids(var_id, var_value, variables=variables): value.append(val) continue for var in variables: - if variables[var]['id'] == var_id: + if variables[var]['alias'] == var_alias: if 'categories' in variables[var]: for cat in variables[var]['categories']: if cat['name'] == val: @@ -598,22 +588,21 @@ def category_ids(var_id, var_value, variables=variables): # variable has no categories, return original # list of values value = var_value - elif isinstance(var_value, str): - for var in variables: + for va, var in variables.items(): # if the variable is a date, don't try to process it's categories - if variables[var]['type'] == 'datetime': + if var['type'] == 'datetime': return var_value - if variables[var]['id'] == var_id and 'categories' in variables[var]: + if va == var_alias and 'categories' in var: found = False - for cat in variables[var]['categories']: + for cat in var['categories']: if cat['name'] == var_value: value = cat['id'] found = True break if not found: raise ValueError("Couldn't find a category id for category %s in filter for variable %s" % (var_value, var)) - elif 'categories' not in variables[var]: + elif 'categories' not in variables[var['alias']]: return var_value else: @@ -623,20 +612,19 @@ def category_ids(var_id, var_value, variables=variables): # special case for multiple_response variables if len(subitems) == 2: _variable, _value = subitems - var_url = _variable.get('variable') + var_alias = _variable.get('var') _value_key = next(iter(_value)) - if _value_key in {'column', "value"} and var_url: - if var_url in var_index and var_index[var_url]['type'] == 'multiple_response': - result = adapt_multiple_response(var_url, _value[_value_key], var_index) + if _value_key in {'column', "value"} and var_alias: + vars_by_alias = {v['alias']: v for _, v in var_index.items()} + if var_alias in vars_by_alias and vars_by_alias[var_alias]['type'] == 'multiple_response': + result = adapt_multiple_response(var_alias, _value[_value_key], vars_by_alias) # handle the multiple response type - _update_values_for_multiple_response(result[0], values, subitems[0], var_index, arrays) + _update_values_for_multiple_response(result[0], values, subitems[0], vars_by_alias, arrays) return result for item in subitems: - if isinstance(item, dict) and 'variable' in item and not isinstance(item["variable"], dict): - var_id = variable_id(item['variable']) - elif isinstance(item, dict) and 'value' in item: - item['value'] = category_ids(var_id, item['value']) + if isinstance(item, dict) and 'value' in item: + item['value'] = category_ids(var_alias, item['value']) _subitems.append(item) return _subitems, True @@ -648,31 +636,12 @@ def _process(obj, variables): subvariables = [] needs_wrap = True - # inspect function, then inspect variable, if multiple_response, - # then change in --> any - if 'function' in obj and 'args' in obj: - if obj['function'] == 'in': - args = obj['args'] - if 'variable' in args[0]: - if isinstance(args[0], dict): - # This is the case of a square bracket subvariable - # array[subvar] in [values] - # In this case, we do not need to do the `in` to `any` - # function conversion, because this subvariable will - # never be of type multiple_response. - pass - else: - try: - if variables.get(args[0]['variable'])['type'] == 'multiple_response': - obj['function'] = 'any' - except TypeError: - raise ValueError("Invalid variable alias '%s'" % args[0]['variable']) - + new_obj = copy.deepcopy(obj) for key, val in obj.items(): if isinstance(val, dict) and "array" not in val: # This is not an array object, then it's a nested ZCL expression # so we need to proceed for nested processing. - obj[key] = _process(val, variables) + new_obj[key] = _process(val, variables) elif isinstance(val, (list, tuple)): subitems = [] for subitem in val: @@ -682,6 +651,13 @@ def _process(obj, variables): arrays.append(subitem.pop('subvariables')) elif 'value' in subitem or 'column' in subitem: values.append(subitem) + elif 'var' in subitem: + var = variables.get(subitem['var']) + if var['type'] in ARRAY_TYPES and 'axes' not in subitem: + # Add info about the fact that the "var" referenced + # variable is an array, so that the validation can + # work properly in the remaindere of the code. + arrays.append(var['subvariables']) subitems.append(subitem) has_value = any( @@ -694,56 +670,26 @@ def _process(obj, variables): has_value = any('column' in item for item in subitems if not is_number(item)) has_variable = any( - 'variable' in item for item in subitems if not is_number(item) + 'var' in item for item in subitems if not is_number(item) ) if has_value and has_variable: subitems, needs_wrap = ensure_category_ids(subitems, values, arrays) - obj[key] = subitems - elif key == 'variable': - if isinstance(val, dict) and "array" in val: - # This is a subvariable reference with this shape: - # {"variable": {"array": array_alias, "subvariable": subvariable_alias}` - array_alias, subvar_alias = val["array"], val["subvariable"] - try: - array_value = variables[array_alias] - except KeyError: - raise ValueError("Invalid variable alias '%s'" % array_alias) - subreferences = array_value["subreferences"] - subvar_map = {sr["alias"]: sv_id for sv_id, sr in subreferences.items()} - array_id = array_value["id"] - try: - subvar_id = subvar_map[subvar_alias] - except KeyError: - raise ValueError("Invalid subvariable `%s` for array '%s'" % (subvariables, array_alias)) - subvar_url = "%svariables/%s/subvariables/%s/" % (base_url, array_id, subvar_id) - obj[key] = subvar_url - else: - # Otherwise a regular variable references {"variable": alias} - var = variables.get(val) - if not var: - raise ValueError("Invalid variable alias '%s'" % val) - - # TODO: We shouldn't stitch URLs together, use the API - if var.get('is_subvar'): - obj[key] = '%svariables/%s/subvariables/%s/' \ - % (base_url, var['parent_id'], var['id']) - else: - obj[key] = '%svariables/%s/' % (base_url, var['id']) - - if var['type'] in ARRAY_TYPES: - subvariables = [] - for subvar_id in var.get('subvariables', []): - # In case the subvar_id comes as a subvariable URL - # we want to only consider the ID bit of the URL - subvar_id = subvar_id.strip("/").split("/")[-1] - subvariables.append( - '%svariables/%s/subvariables/%s/' - % (base_url, var['id'], subvar_id) - ) + new_obj[key] = subitems + elif key == 'var': + var = variables.get(val) + if not var: + raise ValueError("Invalid variable alias '%s'" % val) + if var.get('is_subvar'): + parents_by_ids = {v['id']: v for v in variables.values() if not v.get('is_subvar')} + parent = parents_by_ids[var['parent_id']] + new_obj[key] = parent['alias'] + new_obj['axes'] = [val] elif key == 'function': op = val + + obj = new_obj if subvariables: obj['subvariables'] = subvariables @@ -790,8 +736,11 @@ def _process(obj, variables): if len(subvariables) == 1: obj['function'] = real_op - obj["args"] = [ - {'variable': subvariables[0]}, + obj['args'] = [ + { + 'var': var['alias'], + 'axes': [var['subreferences'][subvariables[0]]['alias']] + }, value ] else: @@ -804,7 +753,7 @@ def _process(obj, variables): [{ 'function': real_op, 'args': [ - {'variable': subvar}, + {'var': var['alias'], 'axes': [var['subreferences'][subvar]['alias']]}, value ] } for subvar in subvariables] @@ -842,34 +791,6 @@ def prettify(expr, ds=None): methods = {m[1]: m[0] for m in CRUNCH_METHOD_MAP.items()} functions = {f[1]: f[0] for f in CRUNCH_FUNC_MAP.items()} - def _resolve_variable(var): - is_url = validate_variable_url(var) - - if not is_url: - return var - elif not isinstance(ds, scrunch.datasets.BaseDataset): - raise Exception( - 'Valid Dataset instance is required to resolve variable urls ' - 'in the expression' - ) - var_resource = ds.resource.session.get(var).payload - var_alias = var_resource.body["alias"] - - # From an arbitrary URL we can detect whether this is a variable or a - # subvariable by checking the adjacent resources linked. A subvariable - # will point to its parent `/subvariables/` catalog and refer to its - # array variable by `.fragments["variable"]`. - is_subvariable = 'parent' in var_resource.catalogs and 'variable' in var_resource.fragments - - if is_subvariable: - # Fetch the array variable - array_url = var_resource.fragments['variable'] - array_var = ds.resource.session.get(array_url).payload - array_alias = array_var.body["alias"] - var_alias = "%s[%s]" % (array_alias, var_alias) - - return var_alias - def _resolve_variables(_expr): new_expr = dict( function=_expr['function'], @@ -879,11 +800,6 @@ def _resolve_variables(_expr): if 'function' in arg: # arg is a function, resolve inner variables new_expr['args'].append(_resolve_variables(arg)) - elif 'variable' in arg: - # arg is a variable, resolve - new_expr['args'].append( - {'variable': _resolve_variable(arg['variable'])} - ) else: # arg is neither a variable or function, pass as is new_expr['args'].append(arg) @@ -942,6 +858,11 @@ def _process(fragment, parent=None): value = [clean_integer(v) for v in value] return value + + if 'var' in fragment: + if 'axes' in fragment: + return "{}[{}]".format(fragment['var'], fragment['axes'][0]) + return fragment['var'] return list(fragment.values())[0] diff --git a/scrunch/tests/test_cubes.py b/scrunch/tests/test_cubes.py index c264b7dd..b3acd5f9 100644 --- a/scrunch/tests/test_cubes.py +++ b/scrunch/tests/test_cubes.py @@ -48,7 +48,7 @@ def test_pass_filter_expression(self, mock_fetch_cube): "function": ">", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/0001/" + "var": "var1_alias" }, {"value": 1}, ], diff --git a/scrunch/tests/test_datasets.py b/scrunch/tests/test_datasets.py index 4fbb31a2..971f0d47 100644 --- a/scrunch/tests/test_datasets.py +++ b/scrunch/tests/test_datasets.py @@ -363,7 +363,7 @@ def test_create_numeric(self, mocked_process): { 'function': '*', 'args': [ - {'variable': 'weekly_rent'}, + {'var': 'weekly_rent'}, {'value': 52} ] }, @@ -553,11 +553,6 @@ def _test_sum(dct, use_fsum=False): if use_fsum and res == 1.0: raise Exception('1.0') - # `sum` fails - with pytest.raises(Exception, match='not 1.0'): - _test_sum(targets[0]['foo']) - - # `fsum` does not with pytest.raises(Exception, match='1.0'): _test_sum(targets[0]['foo'], use_fsum=True) @@ -595,7 +590,7 @@ def test_apply_exclusion(self): 'expression': { 'function': '!=', 'args': [ - {'variable': var.url}, # Crunch needs variable URLs! + {'var': 'var1_alias'}, {'value': 0} ] } @@ -631,7 +626,7 @@ def test_gt(self): 'expression': { 'function': '>', 'args': [ - {'variable': var.url}, + {'var': 'var1_alias'}, {'value': 5} ] } @@ -648,7 +643,7 @@ def test_in(self): "expression": { "function": "in", "args": [ - {"variable": var.url}, + {"var": "var1_alias"}, {"value": [32766]} ] } @@ -666,7 +661,7 @@ def test_in_multiple(self): "expression": { "function": "in", "args": [ - {"variable": var.url}, + {"var": "var1_alias"}, {"value": [32766, 32767]} ] } @@ -707,7 +702,7 @@ def test_not_and(self): "function": "in", "args": [ { - "variable": var1.url + "var": "disposition" }, { "value": [ @@ -721,7 +716,7 @@ def test_not_and(self): "function": "==", "args": [ { - "variable": var2.url + "var": "exit_status" }, { "value": 0 @@ -747,7 +742,7 @@ def test_any(self): "function": "in", "args": [ { - "variable": var.url + "var": "var1_alias" }, { "value": [ @@ -774,7 +769,7 @@ def test_not_any(self): "function": "in", "args": [ { - "variable": var.url + "var": "var1_alias" }, { "value": [ @@ -800,7 +795,7 @@ def test_any_multiple(self): "function": "in", "args": [ { - "variable": var.url + "var": "var1_alias" }, { "value": [ @@ -824,7 +819,7 @@ def test_all(self): "expression": { "args": [ { - "variable": var.url + "var": "var1_alias" }, { "value": [32767] @@ -850,7 +845,7 @@ def test_not_all(self): "function": "all", "args": [ { - "variable": var.url + "var": "var1_alias" }, { "value": [ @@ -877,7 +872,7 @@ def test_all_or_all(self): { "args": [ { - "variable": var.url + "var": "var1_alias" }, { "value": [ @@ -890,7 +885,7 @@ def test_all_or_all(self): { "args": [ { - "variable": var.url + "var": "var1_alias" }, { "value": [ @@ -922,7 +917,7 @@ def test_not_all_or_all(self): { "args": [ { - "variable": var.url + "var": "var1_alias" }, { "value": [ @@ -935,7 +930,7 @@ def test_not_all_or_all(self): { "args": [ { - "variable": var.url + "var": "var1_alias" }, { "value": [ @@ -965,7 +960,7 @@ def test_duplicates(self): "function": "duplicates", "args": [ { - "variable": var.url + "var": "var1_alias" } ] } @@ -984,7 +979,7 @@ def test_valid(self): "function": "is_valid", "args": [ { - "variable": var.url + "var": "var1_alias" } ] } @@ -1004,7 +999,7 @@ def test_not_valid(self): { "args": [ { - "variable": var.url + "var": "var1_alias" } ], "function": "is_valid" @@ -1026,7 +1021,7 @@ def test_missing(self): "expression": { "args": [ { - "variable": var.url + "var": "var1_alias" } ], "function": "is_missing" @@ -1049,7 +1044,7 @@ def test_not_missing(self): "function": "is_missing", "args": [ { - "variable": var.url + "var": "var1_alias" } ] } @@ -1069,7 +1064,7 @@ def test_equal(self): "expression": { "args": [ { - "variable": var.url + "var": "var1_alias" }, { "value": 1 @@ -1110,7 +1105,7 @@ def test_nested(self): { "args": [ { - "variable": var1.url + "var": "disposition" }, { "value": 1 @@ -1125,7 +1120,7 @@ def test_nested(self): { "args": [ { - "variable": var2.url + "var": "exit_status" } ], "function": "is_valid" @@ -1136,7 +1131,7 @@ def test_nested(self): { "args": [ { - "variable": var2.url + "var": "exit_status" }, { "value": 1 @@ -1157,7 +1152,7 @@ def test_nested(self): { "args": [ { - "variable": var1.url + "var": "disposition" }, { "value": 0 @@ -1168,7 +1163,7 @@ def test_nested(self): { "args": [ { - "variable": var2.url + "var": "exit_status" }, { "value": 0 @@ -1184,7 +1179,7 @@ def test_nested(self): { "args": [ { - "variable": var1.url + "var": "disposition" }, { "value": 0 @@ -1195,7 +1190,7 @@ def test_nested(self): { "args": [ { - "variable": var2.url + "var": "exit_status" }, { "value": 1 @@ -1223,7 +1218,7 @@ def test_dict_expr(self): expr = { "args": [ { - "variable": "http://test.crunch.io/api/datasets/123/variables/0002/" + "var": "var1_alias" }, { "value": 1 @@ -2108,8 +2103,8 @@ def prepare_ds(self): def test_recode_w_fill(self): ds, session = self.prepare_ds() responses = [ - {"case": "var_a == 1", "variable": "var_a"}, - {"case": "var_b == 1", "variable": "var_b"} + {"case": "var_a == 1", "var": "var_a"}, + {"case": "var_b == 1", "var": "var_b"} ] # This is what we want to test for! @@ -2135,14 +2130,14 @@ def test_recode_w_fill(self): { "function": "==", "args": [ - {"variable": "http://host/api/projects/abc/variables/001/"}, + {"var": "var_a"}, {"value": 1} ] }, { "function": "==", "args": [ - {"variable": "http://host/api/projects/abc/variables/002/"}, + {"var": "var_b"}, {"value": 1} ] } @@ -2154,8 +2149,8 @@ def test_recode_w_fill(self): case_expr, { "map": { - "1": {"variable": "001"}, - "2": {"variable": "002"} + "1": {"var": "var_a"}, + "2": {"var": "var_b"} } } ], @@ -2175,7 +2170,7 @@ def test_recode_w_fill(self): def test_else_code(self): ds, session = self.prepare_ds() responses = [ - {"case": "var_a == 1", "variable": "var_a"}, + {"case": "var_a == 1", "var": "var_a"}, {"case": "else", "missing": True, "name": "Not Asked", "id": 99} ] @@ -2202,7 +2197,7 @@ def test_else_code(self): { "function": "==", "args": [ - {"variable": "http://host/api/projects/abc/variables/001/"}, + {"var": "var_a"}, {"value": 1} ] }, @@ -2214,7 +2209,7 @@ def test_else_code(self): case_expr, { "map": { - "1": {"variable": "001"}, + "1": {"var": "var_a"}, } } ], @@ -2232,8 +2227,8 @@ def test_else_code(self): def test_else_var(self): ds, session = self.prepare_ds() responses = [ - {"case": "var_a == 1", "variable": "var_a"}, - {"case": "else", "variable": "var_b"} + {"case": "var_a == 1", "var": "var_a"}, + {"case": "else", "var": "var_b"} ] # This is what we want to test for! @@ -2259,7 +2254,7 @@ def test_else_var(self): { "function": "==", "args": [ - {"variable": "http://host/api/projects/abc/variables/001/"}, + {"var": "var_a"}, {"value": 1} ] }, @@ -2271,8 +2266,8 @@ def test_else_var(self): case_expr, { "map": { - "1": {"variable": "001"}, - "-1": {"variable": "002"} + "1": {"var": "var_a"}, + "-1": {"var": "var_b"} } } ], @@ -2365,7 +2360,7 @@ def test_recode_single_categorical(self): }, { 'function': '>', 'args': [ - {'variable': 'https://test.crunch.io/api/datasets/123456/variables/001/'}, + {'var': 'var_a'}, {'value': 5} ] }, { @@ -2373,12 +2368,12 @@ def test_recode_single_categorical(self): 'args': [{ 'function': '<', 'args': [ - {'variable': 'https://test.crunch.io/api/datasets/123456/variables/002/'}, + {'var': 'var_b'}, {'value': 10} ]}, { 'function': 'in', 'args': [ - {'variable': 'https://test.crunch.io/api/datasets/123456/variables/003/'}, + {'var': 'var_c'}, {'value': [1, 2, 3]} ] }] @@ -2387,7 +2382,7 @@ def test_recode_single_categorical(self): 'args': [{ 'function': '==', 'args': [ - {'variable': 'https://test.crunch.io/api/datasets/123456/variables/004/'}, + {'var': 'gender'}, {'value': 1} ] }, { @@ -2395,13 +2390,13 @@ def test_recode_single_categorical(self): 'args': [{ 'function': '>=', 'args': [ - {'variable': 'https://test.crunch.io/api/datasets/123456/variables/005/'}, + {'var': 'age'}, {'value': 16} ] }, { 'function': '<=', 'args': [ - {'variable': 'https://test.crunch.io/api/datasets/123456/variables/005/'}, + {'var': 'age'}, {'value': 24} ] }] @@ -2494,7 +2489,7 @@ def test_recode_multiple_response(self): # 'var_a > 5' 'function': '>', 'args': [ - {'variable': 'https://test.crunch.io/api/datasets/123456/variables/%s/' % variables['var_a']['id']}, + {'var': 'var_a'}, {'value': 5} ] }] @@ -2522,13 +2517,13 @@ def test_recode_multiple_response(self): 'args': [{ 'function': '<', 'args': [ - {'variable': 'https://test.crunch.io/api/datasets/123456/variables/%s/' % variables['var_b']['id']}, + {'var': 'var_b'}, {'value': 10} ] }, { 'function': 'in', 'args': [ - {'variable': 'https://test.crunch.io/api/datasets/123456/variables/%s/' % variables['var_c']['id']}, + {'var': 'var_c'}, {'value': [1, 2, 3]} ] }] @@ -2556,15 +2551,15 @@ def test_recode_multiple_response(self): 'function': 'and', 'args': [{ 'function': '==', - 'args': [{'variable': 'https://test.crunch.io/api/datasets/123456/variables/%s/' % variables['gender']['id']}, {'value': 1}] + 'args': [{'var': 'gender'}, {'value': 1}] }, { 'function': 'and', 'args': [{ 'function': '>=', - 'args': [{'variable': 'https://test.crunch.io/api/datasets/123456/variables/%s/' % variables['age']['id']}, {'value': 16}] + 'args': [{'var': 'age'}, {'value': 16}] }, { 'function': '<=', - 'args': [{'variable': 'https://test.crunch.io/api/datasets/123456/variables/%s/' % variables['age']['id']}, {'value': 24}] + 'args': [{'var': 'age'}, {'value': 24}] }] }] }] @@ -2681,7 +2676,7 @@ def test_create_categorical_else_case(self): "function": "==", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/var_a/" + "var": "var_a" }, { "value": 1 @@ -2692,7 +2687,7 @@ def test_create_categorical_else_case(self): "function": "==", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/var_b/" + "var": "var_b" }, { "value": 1 @@ -2709,7 +2704,7 @@ def test_create_categorical_else_case(self): "function": "==", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/var_a/" + "var": "var_a" }, { "value": 1 @@ -2725,7 +2720,7 @@ def test_create_categorical_else_case(self): "function": "==", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/var_b/" + "var": "var_b" }, { "value": 1 @@ -2818,7 +2813,7 @@ def test_create_2_multiple_response_else_case(self): "function": "==", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 20 @@ -2839,7 +2834,7 @@ def test_create_2_multiple_response_else_case(self): "function": "==", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 50 @@ -2866,7 +2861,7 @@ def test_create_2_multiple_response_else_case(self): "function": "==", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 20 @@ -2882,7 +2877,7 @@ def test_create_2_multiple_response_else_case(self): "function": "==", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 50 @@ -3008,7 +3003,7 @@ def test_create_3_multiple_response_else_case(self): "function": "==", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 21 @@ -3022,7 +3017,7 @@ def test_create_3_multiple_response_else_case(self): "function": "is_missing", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" } ] } @@ -3040,7 +3035,7 @@ def test_create_3_multiple_response_else_case(self): "function": "==", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 21 @@ -3056,7 +3051,7 @@ def test_create_3_multiple_response_else_case(self): "function": "is_missing", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" } ] } @@ -3068,7 +3063,7 @@ def test_create_3_multiple_response_else_case(self): "function": "is_missing", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" } ] } @@ -3089,7 +3084,7 @@ def test_create_3_multiple_response_else_case(self): "function": "==", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 51 @@ -3103,7 +3098,7 @@ def test_create_3_multiple_response_else_case(self): "function": "is_missing", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" } ] } @@ -3121,7 +3116,7 @@ def test_create_3_multiple_response_else_case(self): "function": "==", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 51 @@ -3137,7 +3132,7 @@ def test_create_3_multiple_response_else_case(self): "function": "is_missing", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" } ] } @@ -3149,7 +3144,7 @@ def test_create_3_multiple_response_else_case(self): "function": "is_missing", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" } ] } @@ -3176,7 +3171,7 @@ def test_create_3_multiple_response_else_case(self): "function": "==", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 21 @@ -3192,7 +3187,7 @@ def test_create_3_multiple_response_else_case(self): "function": "==", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 51 @@ -3210,7 +3205,7 @@ def test_create_3_multiple_response_else_case(self): "function": ">", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 100 @@ -3231,7 +3226,7 @@ def test_create_3_multiple_response_else_case(self): "function": "==", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 21 @@ -3242,7 +3237,7 @@ def test_create_3_multiple_response_else_case(self): "function": "==", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 51 @@ -3258,7 +3253,7 @@ def test_create_3_multiple_response_else_case(self): "function": ">", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 100 @@ -3273,7 +3268,7 @@ def test_create_3_multiple_response_else_case(self): "function": ">", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 100 @@ -3389,7 +3384,7 @@ def test_create_categorical_missing_case(self): "function": ">", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 40 @@ -3403,7 +3398,7 @@ def test_create_categorical_missing_case(self): "function": "is_missing", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" } ] } @@ -3421,7 +3416,7 @@ def test_create_categorical_missing_case(self): "function": ">", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 40 @@ -3437,7 +3432,7 @@ def test_create_categorical_missing_case(self): "function": "is_missing", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" } ] } @@ -3449,7 +3444,7 @@ def test_create_categorical_missing_case(self): "function": "is_missing", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" } ] } @@ -3470,7 +3465,7 @@ def test_create_categorical_missing_case(self): "function": "<=", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 40 @@ -3484,7 +3479,7 @@ def test_create_categorical_missing_case(self): "function": "is_missing", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" } ] } @@ -3502,7 +3497,7 @@ def test_create_categorical_missing_case(self): "function": "<=", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" }, { "value": 40 @@ -3518,7 +3513,7 @@ def test_create_categorical_missing_case(self): "function": "is_missing", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" } ] } @@ -3530,7 +3525,7 @@ def test_create_categorical_missing_case(self): "function": "is_missing", "args": [ { - "variable": "https://test.crunch.io/api/datasets/123456/variables/age/" + "var": "age" } ] } @@ -3627,14 +3622,14 @@ def test_derive_multiple_response(self): { 'function': '==', 'args': [ - {'variable': 'https://test.crunch.io/api/datasets/123456/variables/%s/' % variables['var_a']['id']}, + {'var': 'var_a'}, {'value': 1} ] }, { 'function': '==', 'args': [ - {'variable': 'https://test.crunch.io/api/datasets/123456/variables/%s/' % variables['var_a']['id']}, + {'var': 'var_a'}, {'value': 2} ] } @@ -3651,14 +3646,14 @@ def test_derive_multiple_response(self): { 'function': '==', 'args': [ - {'variable': 'https://test.crunch.io/api/datasets/123456/variables/%s/' % variables['var_b']['id']}, + {'var': 'var_b'}, {'value': 1} ] }, { 'function': '==', 'args': [ - {'variable': 'https://test.crunch.io/api/datasets/123456/variables/%s/' % variables['var_b']['id']}, + {'var': 'var_b'}, {'value': 2} ] } @@ -3675,14 +3670,14 @@ def test_derive_multiple_response(self): { 'function': '==', 'args': [ - {'variable': 'https://test.crunch.io/api/datasets/123456/variables/%s/' % variables['var_c']['id']}, + {'var': 'var_c'}, {'value': 1} ] }, { 'function': '==', 'args': [ - {'variable': 'https://test.crunch.io/api/datasets/123456/variables/%s/' % variables['var_c']['id']}, + {'var': 'var_c'}, {'value': 2} ] } @@ -6371,7 +6366,7 @@ def test_add_filter(self, filters): 'expression': { 'function': '!=', 'args': [ - {'variable': var.url}, + {'var': var.alias}, {'value': 0} ] } @@ -6479,7 +6474,7 @@ def test_add_multitable(self, multitables): 'template': [ { 'query': [ - {'variable': 'https://test.crunch.io/api/datasets/123456/variables/0001/'} + {'var': 'var1_alias'} ] } ] @@ -6686,7 +6681,7 @@ def test_append_with_filter(self): "function": ">", "args": [ { - 'variable': 'https://test.crunch.io/api/datasets/123456/variables/var_d/' + 'var': 'endtime' }, { "value": "2024-06-03T22:53:52.393" diff --git a/scrunch/tests/test_expressions.py b/scrunch/tests/test_expressions.py index 7698e9ed..d212ebd1 100644 --- a/scrunch/tests/test_expressions.py +++ b/scrunch/tests/test_expressions.py @@ -106,15 +106,15 @@ def test_process_nested(self): { 'function': '==', 'args': [ - {'variable': 'http://host.com/api/datasets/abc123/variables/0001/'}, + {'var': 'identity'}, {'value': 1} ] }, { 'function': '<=', 'args': [ - {'variable': 'http://host.com/api/datasets/abc123/variables/0002/'}, - {'variable': 'http://host.com/api/datasets/abc123/variables/0003/'} + {'var': 'caseid'}, + {'var': 'surveyid'} ] } ] @@ -122,7 +122,7 @@ def test_process_nested(self): { 'function': '>=', 'args': [ - {'variable': 'http://host.com/api/datasets/abc123/variables/0001/'}, + {'var': 'identity'}, {'value': 2} ] } @@ -160,7 +160,7 @@ def test_parse_equal_int(self): 'function': '==', 'args': [ { - 'variable': 'age' + 'var': 'age' }, { 'value': 1 @@ -178,7 +178,7 @@ def test_parse_equal_int(self): 'value': 1 }, { - 'variable': 'age' + 'var': 'age' } ] } @@ -190,7 +190,7 @@ def test_parse_equal_string(self): 'function': '==', 'args': [ { - 'variable': 'name' + 'var': 'name' }, { 'value': 'John Doe' @@ -208,7 +208,7 @@ def test_parse_equal_string(self): 'value': 'John Doe' }, { - 'variable': 'name' + 'var': 'name' } ] } @@ -220,7 +220,7 @@ def test_parse_notequal_int(self): 'function': '!=', 'args': [ { - 'variable': 'age' + 'var': 'age' }, { 'value': 1 @@ -238,7 +238,7 @@ def test_parse_notequal_int(self): 'value': 1 }, { - 'variable': 'age' + 'var': 'age' } ] } @@ -250,7 +250,7 @@ def test_parse_notequal_string(self): 'function': '!=', 'args': [ { - 'variable': 'name' + 'var': 'name' }, { 'value': 'John Doe' @@ -268,7 +268,7 @@ def test_parse_notequal_string(self): 'value': 'John Doe' }, { - 'variable': 'name' + 'var': 'name' } ] } @@ -280,7 +280,7 @@ def test_parse_less_than(self): 'function': '<', 'args': [ { - 'variable': 'caseid' + 'var': 'caseid' }, { 'value': 1234 @@ -298,7 +298,7 @@ def test_parse_less_than(self): 'value': 1234 }, { - 'variable': 'caseid' + 'var': 'caseid' } ] } @@ -310,7 +310,7 @@ def test_parse_less_than_equal(self): 'function': '<=', 'args': [ { - 'variable': 'caseid' + 'var': 'caseid' }, { 'value': 1234 @@ -328,7 +328,7 @@ def test_parse_less_than_equal(self): 'value': 1234 }, { - 'variable': 'caseid' + 'var': 'caseid' } ] } @@ -340,7 +340,7 @@ def test_parse_greater_than(self): 'function': '>', 'args': [ { - 'variable': 'caseid' + 'var': 'caseid' }, { 'value': 1234 @@ -358,7 +358,7 @@ def test_parse_greater_than(self): 'value': 1234 }, { - 'variable': 'caseid' + 'var': 'caseid' } ] } @@ -370,7 +370,7 @@ def test_parse_greater_than_equal(self): 'function': '>=', 'args': [ { - 'variable': 'caseid' + 'var': 'caseid' }, { 'value': 1234 @@ -388,7 +388,7 @@ def test_parse_greater_than_equal(self): 'value': 1234 }, { - 'variable': 'caseid' + 'var': 'caseid' } ] } @@ -400,10 +400,10 @@ def test_parse_compare_variable_against_another_variable(self): 'function': '==', 'args': [ { - 'variable': 'starttdate' + 'var': 'starttdate' }, { - 'variable': 'arrivedate' + 'var': 'arrivedate' } ] } @@ -414,10 +414,10 @@ def test_parse_compare_variable_against_another_variable(self): 'function': '!=', 'args': [ { - 'variable': 'starttdate' + 'var': 'starttdate' }, { - 'variable': 'arrivedate' + 'var': 'arrivedate' } ] } @@ -428,10 +428,10 @@ def test_parse_compare_variable_against_another_variable(self): 'function': '<', 'args': [ { - 'variable': 'starttdate' + 'var': 'starttdate' }, { - 'variable': 'arrivedate' + 'var': 'arrivedate' } ] } @@ -442,10 +442,10 @@ def test_parse_compare_variable_against_another_variable(self): 'function': '<=', 'args': [ { - 'variable': 'starttdate' + 'var': 'starttdate' }, { - 'variable': 'arrivedate' + 'var': 'arrivedate' } ] } @@ -456,10 +456,10 @@ def test_parse_compare_variable_against_another_variable(self): 'function': '>', 'args': [ { - 'variable': 'starttdate' + 'var': 'starttdate' }, { - 'variable': 'arrivedate' + 'var': 'arrivedate' } ] } @@ -470,10 +470,10 @@ def test_parse_compare_variable_against_another_variable(self): 'function': '>=', 'args': [ { - 'variable': 'starttdate' + 'var': 'starttdate' }, { - 'variable': 'arrivedate' + 'var': 'arrivedate' } ] } @@ -491,7 +491,7 @@ def test_parse_multiple_boolean_conditions(self): 'function': '==', 'args': [ { - 'variable': 'identity' + 'var': 'identity' }, { 'value': 1 @@ -502,10 +502,10 @@ def test_parse_multiple_boolean_conditions(self): 'function': '<=', 'args': [ { - 'variable': 'caseid' + 'var': 'caseid' }, { - 'variable': 'surveyid' + 'var': 'surveyid' } ] } @@ -515,7 +515,7 @@ def test_parse_multiple_boolean_conditions(self): 'function': '>=', 'args': [ { - 'variable': 'identity' + 'var': 'identity' }, { 'value': 2 @@ -532,7 +532,7 @@ def test_parse_value_in_list(self): 'function': 'in', 'args': [ { - 'variable': 'web_browser' + 'var': 'web_browser' }, { 'value': ['abc', 'dfg', 'hij'] @@ -547,7 +547,7 @@ def test_parse_value_in_list(self): 'function': 'in', 'args': [ { - 'variable': 'web_browser' + 'var': 'web_browser' }, { 'value': ['abc', 'dfg', 'hij'] @@ -561,7 +561,7 @@ def test_parse_float_value_in_list(self): 'function': 'in', 'args': [ { - 'variable': 'country_cat' + 'var': 'country_cat' }, { 'value': [1.0] @@ -577,7 +577,7 @@ def test_parse_integer_value_in_list(self): 'function': 'in', 'args': [ { - 'variable': 'country_cat' + 'var': 'country_cat' }, { 'value': [1] @@ -592,7 +592,7 @@ def test_r_in(self): expected_expr_obj = { 'args': [ - {'variable': 'q1'}, + {'var': 'q1'}, {'value': [1, 2, 4, 5, 6, 7, 10, 11, 12]} ], 'function': 'in' @@ -616,7 +616,7 @@ def test_parse_value_not_in_list(self): 'function': 'in', 'args': [ { - 'variable': 'country' + 'var': 'country' }, { 'value': [1, 2, 3] @@ -636,7 +636,7 @@ def test_parse_value_not_in_list(self): 'function': 'in', 'args': [ { - 'variable': 'country' + 'var': 'country' }, { 'value': [1, 2, 3] @@ -659,7 +659,7 @@ def test_parse_sample_rule_1(self): 'function': '==', 'args': [ { - 'variable': 'disposition' + 'var': 'disposition' }, { 'value': 0 @@ -670,7 +670,7 @@ def test_parse_sample_rule_1(self): 'function': '==', 'args': [ { - 'variable': 'exit_status' + 'var': 'exit_status' }, { 'value': 0 @@ -687,7 +687,7 @@ def test_parse_any(self): 'function': 'any', 'args': [ { - 'variable': 'Q2' + 'var': 'Q2' }, { 'value': [1, 2, 3] @@ -701,7 +701,7 @@ def test_parse_any(self): 'function': 'any', 'args': [ { - 'variable': 'Q2' + 'var': 'Q2' }, { 'value': [1, 2, 3] @@ -724,7 +724,7 @@ def test_parse_all(self): 'function': 'all', 'args': [ { - 'variable': 'Q2' + 'var': 'Q2' }, { 'value': [1, 2, 3] @@ -738,7 +738,7 @@ def test_parse_all(self): 'function': 'all', 'args': [ { - 'variable': 'Q2' + 'var': 'Q2' }, { 'value': [1, 2, 3] @@ -771,7 +771,7 @@ def test_parse_sample_rule_2_complex(self): 'function': '==', 'args': [ { - 'variable': 'disposition' + 'var': 'disposition' }, { 'value': 0 @@ -782,7 +782,7 @@ def test_parse_sample_rule_2_complex(self): 'function': '==', 'args': [ { - 'variable': 'exit_status' + 'var': 'exit_status' }, { 'value': 1 @@ -797,7 +797,7 @@ def test_parse_sample_rule_2_complex(self): 'function': '==', 'args': [ { - 'variable': 'disposition' + 'var': 'disposition' }, { 'value': 0 @@ -808,7 +808,7 @@ def test_parse_sample_rule_2_complex(self): 'function': '==', 'args': [ { - 'variable': 'exit_status' + 'var': 'exit_status' }, { 'value': 0 @@ -825,7 +825,7 @@ def test_mr_any_subvar(self): assert parsed_zcl_expr == { 'function': 'any', 'args': [ - {'variable': 'MyMrVar'}, + {'var': 'MyMrVar'}, {'column': ['subvar1', 'subvar2']} ] } @@ -836,7 +836,7 @@ def test_mr_all_subvar(self): assert expr_obj == { 'function': 'all', 'args': [ - {'variable': 'MyMrVar'}, + {'var': 'MyMrVar'}, {'column': ['subvar1', 'subvar2']} ] } @@ -847,7 +847,7 @@ def test_mr_in_subvar(self): assert expr_obj == { 'function': 'in', 'args': [ - {'variable': 'MyMrVar'}, + {'var': 'MyMrVar'}, {'column': ['subvar1', 'subvar2']} ] } @@ -864,7 +864,7 @@ def test_parse_sample_any(self): 'function': 'any', 'args': [ { - 'variable': 'CompanyTurnover' + 'var': 'CompanyTurnover' }, { 'value': [99] @@ -878,7 +878,7 @@ def test_parse_sample_any(self): 'function': 'any', 'args': [ { - 'variable': 'sector' + 'var': 'sector' }, { 'value': [2, 3, 98, 99] @@ -896,7 +896,7 @@ def test_parse_negated_expr(self): 'function': '==', 'args': [ { - 'variable': 'age' + 'var': 'age' }, { 'value': 1 @@ -916,7 +916,7 @@ def test_parse_negated_method_call(self): 'function': 'any', 'args': [ { - 'variable': 'Q2' + 'var': 'Q2' }, { 'value': [1, 2, 3] @@ -935,7 +935,7 @@ def test_parse_negated_method_call(self): 'function': 'all', 'args': [ { - 'variable': 'Q2' + 'var': 'Q2' }, { 'value': [1, 2, 3] @@ -952,7 +952,7 @@ def test_parse_duplicates_method(self): 'function': 'duplicates', 'args': [ { - 'variable': 'identity' + 'var': 'identity' } ] } @@ -967,7 +967,7 @@ def test_parse_duplicates_method(self): 'function': 'duplicates', 'args': [ { - 'variable': 'identity' + 'var': 'identity' } ] } @@ -993,7 +993,7 @@ def test_multiple_and_or(self): 'args': [ { 'args': [ - {'variable': 'age'}, + {'var': 'age'}, {'value': 1} ], 'function': '==' @@ -1002,14 +1002,14 @@ def test_multiple_and_or(self): 'args': [ { 'args': [ - {'variable': 'test'}, + {'var': 'test'}, {'value': 3} ], 'function': '==' }, { 'args': [ - {'variable': 'myop'}, + {'var': 'myop'}, {'value': 'age'} ], 'function': '==' @@ -1031,14 +1031,14 @@ def test_arithmetic_operations(self): 'args': [ { 'args': [ - {'variable': 'var1'}, + {'var': 'var1'}, {'value': 3} ], 'function': '+' }, { 'args': [ - {'variable': 'var2'}, + {'var': 'var2'}, {'value': 2} ], 'function': '-'} @@ -1049,14 +1049,14 @@ def test_arithmetic_operations(self): 'args': [ { 'args': [ - {'variable': 'var3'}, + {'var': 'var3'}, {'value': 1} ], 'function': '/' }, { 'args': [ - {'variable': 'var4'}, + {'var': 'var4'}, {'value': 10} ], 'function': '*'} @@ -1077,7 +1077,7 @@ def test_arithmetic_operator_presedence(self): 'args': [ { 'args': [ - {'variable': 'var1'}, + {'var': 'var1'}, {'value': 10} ], 'function': '*' @@ -1092,7 +1092,7 @@ def test_arithmetic_operator_presedence(self): ], 'function': '+' }, - {'variable': 'var2'} + {'var': 'var2'} ], 'function': '==' } @@ -1156,7 +1156,7 @@ def test_multiple_arithmetic_operations_with_variable(self): "function": "*", "args": [ { - "variable": "weekly_rent" + "var": "weekly_rent" }, { "value": 52 @@ -1177,7 +1177,7 @@ def test_parse_helper_functions(self): 'function': 'is_valid', 'args': [ { - 'variable': 'birthyear' + 'var': 'birthyear' } ] } @@ -1188,7 +1188,7 @@ def test_parse_helper_functions(self): 'function': 'is_missing', 'args': [ { - 'variable': 'birthyear' + 'var': 'birthyear' } ] } @@ -1203,7 +1203,7 @@ def test_parse_helper_functions(self): 'function': 'is_valid', 'args': [ { - 'variable': 'birthyear' + 'var': 'birthyear' } ] } @@ -1219,7 +1219,7 @@ def test_parse_helper_functions(self): 'function': 'is_missing', 'args': [ { - 'variable': 'birthyear' + 'var': 'birthyear' } ] } @@ -1236,7 +1236,7 @@ def test_parse_helper_functions(self): 'function': 'is_valid', 'args': [ { - 'variable': 'birthyear' + 'var': 'birthyear' } ] }, @@ -1244,7 +1244,7 @@ def test_parse_helper_functions(self): 'function': 'is_valid', 'args': [ { - 'variable': 'birthmonth' + 'var': 'birthmonth' } ] } @@ -1260,7 +1260,7 @@ def test_parse_helper_functions(self): 'function': 'is_missing', 'args': [ { - 'variable': 'birthyear' + 'var': 'birthyear' } ] }, @@ -1268,7 +1268,7 @@ def test_parse_helper_functions(self): 'function': 'is_missing', 'args': [ { - 'variable': 'birthmonth' + 'var': 'birthmonth' } ] } @@ -1288,7 +1288,7 @@ def test_parse_helper_functions(self): 'function': 'is_valid', 'args': [ { - 'variable': 'birthyear' + 'var': 'birthyear' } ] }, @@ -1296,7 +1296,7 @@ def test_parse_helper_functions(self): 'function': 'is_valid', 'args': [ { - 'variable': 'birthmonth' + 'var': 'birthmonth' } ] } @@ -1317,7 +1317,7 @@ def test_parse_helper_functions(self): 'function': 'is_missing', 'args': [ { - 'variable': 'birthyear' + 'var': 'birthyear' } ] }, @@ -1325,7 +1325,7 @@ def test_parse_helper_functions(self): 'function': 'is_missing', 'args': [ { - 'variable': 'birthmonth' + 'var': 'birthmonth' } ] } @@ -1344,7 +1344,7 @@ def test_parse_helper_functions(self): 'function': '<', 'args': [ { - 'variable': 'caseid' + 'var': 'caseid' }, { 'value': 12345 @@ -1358,7 +1358,7 @@ def test_parse_helper_functions(self): 'function': 'is_missing', 'args': [ { - 'variable': 'birthyear' + 'var': 'birthyear' } ] }, @@ -1366,7 +1366,7 @@ def test_parse_helper_functions(self): 'function': 'is_missing', 'args': [ { - 'variable': 'birthmonth' + 'var': 'birthmonth' } ] } @@ -1382,17 +1382,17 @@ def test_multiple_missing_valid(self): assert expr_obj == { 'args': [ { - 'args': [{'variable': 'year'}], + 'args': [{'var': 'year'}], 'function': 'is_missing' }, { 'args': [ { - 'args': [{'variable': 'month'}], + 'args': [{'var': 'month'}], 'function': 'is_missing' }, { - 'args': [{'variable': 'age'}], + 'args': [{'var': 'age'}], 'function': 'is_missing' } ], @@ -1407,23 +1407,23 @@ def test_multiple_missing_valid(self): assert expr_obj == { 'args': [ { - 'args': [{'variable': 'year'}], + 'args': [{'var': 'year'}], 'function': 'is_valid' }, { 'args': [ { - 'args': [{'variable': 'month'}], + 'args': [{'var': 'month'}], 'function': 'is_valid' }, { 'args': [ { - 'args': [{'variable': 'age'}], + 'args': [{'var': 'age'}], 'function': 'is_valid' }, { - 'args': [{'variable': 'gender'}], + 'args': [{'var': 'gender'}], 'function': 'is_valid' } ], @@ -1446,7 +1446,7 @@ def test_parse_not_a_in_b(self): 'function': 'in', 'args': [ { - 'variable': 'a' + 'var': 'a' }, { 'value': [1, 2, 3] @@ -1465,7 +1465,7 @@ def test_parse_not_a_in_b(self): 'function': 'in', 'args': [ { - 'variable': 'a' + 'var': 'a' }, { 'value': [1, 2, 3] @@ -1484,7 +1484,7 @@ def test_parse_not_a_in_b(self): 'function': 'in', 'args': [ { - 'variable': 'a' + 'var': 'a' }, { 'value': [1, 2, 3] @@ -1496,7 +1496,7 @@ def test_parse_not_a_in_b(self): def test_parse_subvariable_brackets(self): expr = "array_alias[subvariable_alias] in [1, 2, 3]" - expr_obj = parse_expr(expr, platonic=False) + expr_obj = parse_expr(expr) assert expr_obj == { 'function': 'in', 'args': [ @@ -1504,11 +1504,11 @@ def test_parse_subvariable_brackets(self): # this is a temporary intern format, so we can use this later # on to convert to URLs appropriately discovering first the # array and then the subvariable - {'variable': {"array": "array_alias", "subvariable": "subvariable_alias"}}, + {'var': "array_alias", "axes": ["subvariable_alias"]}, {'value': [1, 2, 3]} ] } - expr_obj = parse_expr(expr, platonic=True) + expr_obj = parse_expr(expr) assert expr_obj == { 'function': 'in', 'args': [ @@ -1523,7 +1523,7 @@ def test_parse_subvariable_brackets(self): def test_parse_platonic_expr(self): expr = """not (array[subvar] or num_val) and other[dimension] and not logical""" - parsed = parse_expr(expr, platonic=True) + parsed = parse_expr(expr) assert parsed == { 'function': 'and', 'args': [ @@ -1650,7 +1650,7 @@ def test_transform_alias_to_var_id(self): 'function': '==', 'args': [ { - 'variable': var_url + 'var': var_alias }, { 'value': 1 @@ -1665,10 +1665,10 @@ def test_adapt_multiple_response_any_subvar(self): var_type = 'multiple_response' var_url = '{}variables/{}/'.format(self.ds_url, var_id) var_categories = [ - {"id": 1, "name": "cat1", "selected": True}, - {"id": 2, "name": "cat2", "selected": True}, - {"id": 3, "name": "cat3", "selected": False}, - ] + {"id": 1, "name": "cat1", "selected": True}, + {"id": 2, "name": "cat2", "selected": True}, + {"id": 3, "name": "cat3", "selected": False}, + ] table_mock = mock.MagicMock(metadata={ var_id: { @@ -1695,7 +1695,7 @@ def test_adapt_multiple_response_any_subvar(self): "name": "Multiple Response", "description": "", "notes": "", - "alias": "mr_variable", + "alias": var_alias, "id": "{}".format(var_id), "type": "multiple_response", "subvariables": [ @@ -1711,10 +1711,11 @@ def test_adapt_multiple_response_any_subvar(self): with mock.patch("scrunch.expressions.get_subvariables_resource") as mock_subvars, mock.patch("scrunch.expressions._get_categories_from_var_index") as categories: categories.return_value = var_categories mock_subvars.return_value = dict(sorted({"subvar1": "001", "subvar2": "002", "subvar3": "003"}.items())) - result, need_wrap = adapt_multiple_response(var_url, values, ds.variables.index) + vars_by_alias = {v['alias']: v for _, v in ds.variables.index.items()} + result, need_wrap = adapt_multiple_response(var_alias, values, vars_by_alias) assert result == [ - {'variable': "{}subvariables/001/".format(var_url), 'column': [1, 2]}, - {'variable': "{}subvariables/002/".format(var_url), 'column': [1, 2]} + {'var': var_alias, 'axes': ['subvar1'], 'column': [1, 2]}, + {'var': var_alias, 'axes': ['subvar2'], 'column': [1, 2]} ] assert need_wrap is True @@ -1920,7 +1921,7 @@ def test_multiple_response_any_process_single_subvariables(self): "name": "Multiple Response", "description": "", "notes": "", - "alias": "mr_variable", + "alias": var_alias, "id": "{}".format(var_id), "type": "multiple_response", "subvariables": [ @@ -1960,7 +1961,8 @@ def test_multiple_response_any_process_single_subvariables(self): 'function': 'in', 'args': [ { - 'variable': 'http://test.crunch.io/api/datasets/123/variables/0001/subvariables/001/' + 'var': 'MyMrVar', + 'axes': ['subvar1'] }, { 'column': [1] @@ -2005,7 +2007,7 @@ def test_multiple_response_any_process_two_subvariables(self): "name": "Multiple Response", "description": "", "notes": "", - "alias": "mr_variable", + "alias": var_alias, "id": "{}".format(var_id), "type": "multiple_response", "subvariables": [ @@ -2047,7 +2049,8 @@ def test_multiple_response_any_process_two_subvariables(self): 'function': 'in', 'args': [ { - 'variable': 'http://test.crunch.io/api/datasets/123/variables/0001/subvariables/001/' + 'var': 'MyMrVar', + 'axes': ['subvar1'] }, { 'column': [1] @@ -2058,7 +2061,8 @@ def test_multiple_response_any_process_two_subvariables(self): 'function': 'in', 'args': [ { - 'variable': 'http://test.crunch.io/api/datasets/123/variables/0001/subvariables/002/' + 'var': 'MyMrVar', + 'axes': ['subvar2'] }, { 'column': [1] @@ -2141,7 +2145,8 @@ def test_multiple_response_subvar_equality(self): 'function': '==', 'args': [ { - 'variable': "{}subvariables/001/".format(var_url), + 'var': var_alias, + 'axes': ['subvar1'] }, { 'value': 1 @@ -2184,7 +2189,8 @@ def test_transform_subvar_alias_to_subvar_id(self): 'function': '==', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[0]) + 'var': var_alias, + 'axes': ['hobbies_1'] }, { 'value': 4 @@ -2217,7 +2223,7 @@ def test_transform_subvar_alias_w_brackets_to_subvar_id(self): # Expression with subvariable indicated by bracket syntax expr = "hobbies_array[hobbies_1] == 4" - parsed_platonic = parse_expr(expr, platonic=True) + parsed_platonic = parse_expr(expr) assert parsed_platonic == { 'function': '==', 'args': [ @@ -2231,7 +2237,7 @@ def test_transform_subvar_alias_w_brackets_to_subvar_id(self): 'function': '==', 'args': [ # Stores a reference to the pieces of the array/subvariable - {"variable": {"array": 'hobbies_array', "subvariable": 'hobbies_1'}}, + {"var": 'hobbies_array', "axes": ['hobbies_1']}, {'value': 4} ] } @@ -2240,14 +2246,14 @@ def test_transform_subvar_alias_w_brackets_to_subvar_id(self): 'function': '==', 'args': [ # Correctly translates into the subvariable URL - {'variable': subvariable_url}, + {'var': 'hobbies_array', 'axes': ['hobbies_1']}, {'value': 4} ] } # Expression with subvariable indicated by bracket syntax expr = "hobbies_array[hobbies_1].any([1, 2])" - parsed_platonic = parse_expr(expr, platonic=True) + parsed_platonic = parse_expr(expr) assert parsed_platonic == { 'function': "any", 'args': [ @@ -2261,7 +2267,7 @@ def test_transform_subvar_alias_w_brackets_to_subvar_id(self): 'function': "any", 'args': [ # Stores a reference to the array/subvairable - {"variable": {"array": 'hobbies_array', "subvariable": 'hobbies_1'}}, + {"var": 'hobbies_array', "axes": ['hobbies_1']}, {'value': [1, 2]} ] } @@ -2270,14 +2276,14 @@ def test_transform_subvar_alias_w_brackets_to_subvar_id(self): 'function': "in", 'args': [ # Still finds the correct subvariable ID under the array URL - {'variable': subvariable_url}, + {'var': 'hobbies_array', 'axes': ['hobbies_1']}, {'value': [1, 2]} ] } # `IN` functions have a bit of a special treatment. expr = "hobbies_array[hobbies_1] in [1]" - parsed_platonic = parse_expr(expr, platonic=True) + parsed_platonic = parse_expr(expr) assert parsed_platonic == { 'function': 'in', 'args': [ @@ -2291,7 +2297,7 @@ def test_transform_subvar_alias_w_brackets_to_subvar_id(self): 'function': 'in', 'args': [ # Stores a reference to the pieces of the array/subvariable - {"variable": {"array": 'hobbies_array', "subvariable": 'hobbies_1'}}, + {"var": 'hobbies_array', "axes": ['hobbies_1']}, {'value': [1]} ] } @@ -2300,7 +2306,7 @@ def test_transform_subvar_alias_w_brackets_to_subvar_id(self): 'function': 'in', 'args': [ # Correctly translates into the subvariable URL - {'variable': subvariable_url}, + {'var': 'hobbies_array', 'axes': ['hobbies_1']}, {'value': [1]} ] } @@ -2330,7 +2336,7 @@ def test_platonic_filter(self): # Expression with subvariable indicated by bracket syntax expr = "hobbies_array[hobbies_1] == 4" - parsed = parse_expr(expr, platonic=True) + parsed = parse_expr(expr) assert parsed == { 'function': '==', 'args': [ @@ -2342,12 +2348,12 @@ def test_platonic_filter(self): expr_obj = process_expr(parsed, ds) assert expr_obj == parsed - parsed = parse_expr(expr, platonic=False) + parsed = parse_expr(expr) assert parsed == { 'function': '==', 'args': [ # Keeps the platonic reference to the subvariable - {"variable": {"array": 'hobbies_array', "subvariable": 'hobbies_1'}}, + {"var": 'hobbies_array', "axes": ['hobbies_1']}, {'value': 4} ] } @@ -2356,7 +2362,7 @@ def test_platonic_filter(self): 'function': '==', 'args': [ # Keeps the platonic reference to the subvariable - {"variable": subvariable_url}, + {"var": 'hobbies_array', "axes": ['hobbies_1']}, {'value': 4} ] } @@ -2394,7 +2400,8 @@ def test_array_expansion_single_subvariable_any(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[0]) + 'var': 'hobbies', + 'axes': ['hobbies_1'] }, { 'value': [32766] @@ -2434,7 +2441,8 @@ def test_array_expansion_single_subvariable_all(self): 'function': '==', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[0]) + 'var': 'hobbies', + 'axes': ['hobbies_1'] }, { 'value': 32766 @@ -2478,7 +2486,8 @@ def test_array_expansion_single_subvariable_not_any(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[0]) + 'var': 'hobbies', + 'axes': ['hobbies_1'] }, { 'value': [32766] @@ -2524,7 +2533,7 @@ def test_array_expansion_single_subvariable_not_all(self): 'function': '==', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[0]) + 'var': 'hobbies', 'axes': ['hobbies_1'] }, { 'value': 32766 @@ -2568,7 +2577,8 @@ def test_array_expansion_single_subvariable_multiple_any(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[0]) + 'var': 'hobbies', + 'axes': ['hobbies_1'] }, { 'value': [32766, 32767] @@ -2647,7 +2657,8 @@ def test_categorical_array_any_expansion_multiple_subvariables(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[0]) + 'var': 'hobbies', + 'axes': ['hobbies_1'], }, { 'value': [32766] @@ -2658,7 +2669,8 @@ def test_categorical_array_any_expansion_multiple_subvariables(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[1]) + 'var': 'hobbies', + 'axes': ['hobbies_2'], }, { 'value': [32766] @@ -2669,7 +2681,8 @@ def test_categorical_array_any_expansion_multiple_subvariables(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[2]) + 'var': 'hobbies', + 'axes': ['hobbies_3'], }, { 'value': [32766] @@ -2680,7 +2693,8 @@ def test_categorical_array_any_expansion_multiple_subvariables(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[3]) + 'var': 'hobbies', + 'axes': ['hobbies_4'], }, { 'value': [32766] @@ -2731,7 +2745,8 @@ def test_categorical_array_all_process_expression(self): 'function': '==', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[0]) + 'var': 'hobbies', + 'axes': ['hobbies_1'], }, { 'value': 32766 @@ -2742,7 +2757,8 @@ def test_categorical_array_all_process_expression(self): 'function': '==', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[1]) + 'var': 'hobbies', + 'axes': ['hobbies_2'], }, { 'value': 32766 @@ -2753,7 +2769,8 @@ def test_categorical_array_all_process_expression(self): 'function': '==', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[2]) + 'var': 'hobbies', + 'axes': ['hobbies_3'], }, { 'value': 32766 @@ -2764,7 +2781,8 @@ def test_categorical_array_all_process_expression(self): 'function': '==', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[3]) + 'var': 'hobbies', + 'axes': ['hobbies_4'], }, { 'value': 32766 @@ -2819,7 +2837,8 @@ def test_categorical_array_not_any_process_expression(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[0]) + 'var': 'hobbies', + 'axes': ['hobbies_1'], }, { 'value': [32766] @@ -2830,7 +2849,8 @@ def test_categorical_array_not_any_process_expression(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[1]) + 'var': 'hobbies', + 'axes': ['hobbies_2'], }, { 'value': [32766] @@ -2841,7 +2861,8 @@ def test_categorical_array_not_any_process_expression(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[2]) + 'var': 'hobbies', + 'axes': ['hobbies_3'], }, { 'value': [32766] @@ -2852,7 +2873,8 @@ def test_categorical_array_not_any_process_expression(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[3]) + 'var': 'hobbies', + 'axes': ['hobbies_4'], }, { 'value': [32766] @@ -2908,7 +2930,8 @@ def test_categorical_array_not_all_process_expression(self): 'function': '==', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[0]) + 'var': 'hobbies', + 'axes': ['hobbies_1'], }, { 'value': 32766 @@ -2919,7 +2942,8 @@ def test_categorical_array_not_all_process_expression(self): 'function': '==', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[1]) + 'var': 'hobbies', + 'axes': ['hobbies_2'], }, { 'value': 32766 @@ -2930,7 +2954,8 @@ def test_categorical_array_not_all_process_expression(self): 'function': '==', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[2]) + 'var': 'hobbies', + 'axes': ['hobbies_3'], }, { 'value': 32766 @@ -2941,7 +2966,8 @@ def test_categorical_array_not_all_process_expression(self): 'function': '==', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[3]) + 'var': 'hobbies', + 'axes': ['hobbies_4'], }, { 'value': 32766 @@ -2995,7 +3021,8 @@ def test_categorical_array_any_multiple_selection_process_expression(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[0]) + 'var': 'hobbies', + 'axes': ['hobbies_1'], }, { 'value': [32766, 32767] @@ -3006,7 +3033,8 @@ def test_categorical_array_any_multiple_selection_process_expression(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[1]) + 'var': 'hobbies', + 'axes': ['hobbies_2'], }, { 'value': [32766, 32767] @@ -3017,7 +3045,8 @@ def test_categorical_array_any_multiple_selection_process_expression(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[2]) + 'var': 'hobbies', + 'axes': ['hobbies_3'], }, { 'value': [32766, 32767] @@ -3028,7 +3057,8 @@ def test_categorical_array_any_multiple_selection_process_expression(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[3]) + 'var': 'hobbies', + 'axes': ['hobbies_4'], }, { 'value': [32766, 32767] @@ -3083,7 +3113,8 @@ def test_categorical_array_not_any_multiple_selection_process_expression(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[0]) + 'var': 'hobbies', + 'axes': ['hobbies_1'], }, { 'value': [32766, 32767] @@ -3094,7 +3125,8 @@ def test_categorical_array_not_any_multiple_selection_process_expression(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[1]) + 'var': 'hobbies', + 'axes': ['hobbies_2'], }, { 'value': [32766, 32767] @@ -3105,7 +3137,8 @@ def test_categorical_array_not_any_multiple_selection_process_expression(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[2]) + 'var': 'hobbies', + 'axes': ['hobbies_3'], }, { 'value': [32766, 32767] @@ -3116,7 +3149,8 @@ def test_categorical_array_not_any_multiple_selection_process_expression(self): 'function': 'in', 'args': [ { - 'variable': '%ssubvariables/%s/' % (var_url, subvariables[3]) + 'var': 'hobbies', + 'axes': ['hobbies_4'], }, { 'value': [32766, 32767] @@ -3167,7 +3201,7 @@ def test_valid_and_missing_funcs_for_arrays(self): 'function': 'all_valid', 'args': [ { - 'variable': var_url + 'var': var_alias } ] } @@ -3181,7 +3215,7 @@ def test_valid_and_missing_funcs_for_arrays(self): 'function': 'all_valid', 'args': [ { - 'variable': var_url + 'var': var_alias } ] } @@ -3194,7 +3228,7 @@ def test_valid_and_missing_funcs_for_arrays(self): 'function': 'is_missing', 'args': [ { - 'variable': var_url + 'var': var_alias } ] } @@ -3208,7 +3242,7 @@ def test_valid_and_missing_funcs_for_arrays(self): 'function': 'is_missing', 'args': [ { - 'variable': var_url + 'var': var_alias } ] } @@ -3245,7 +3279,7 @@ def test_label_expression_single(self): 'function': '==', 'args': [ { - 'variable': var_url + 'var': var_alias }, { 'value': 1 @@ -3287,7 +3321,7 @@ def test_label_expression_list(self): 'function': 'in', 'args': [ { - 'variable': var_url + 'var': 'hobbies' }, { 'value': [1, 2] @@ -3329,7 +3363,7 @@ def test_label_expression_tuple(self): 'function': 'in', 'args': [ { - 'variable': var_url + 'var': var_alias }, { 'value': [1, 2] @@ -3372,7 +3406,7 @@ def test_any_categorical_var(self): 'function': 'in', 'args': [ { - 'variable': var_url + 'var': var_alias }, { 'value': [1] @@ -3414,7 +3448,7 @@ def test_in_expression_list_integer(self): 'function': 'in', 'args': [ { - 'variable': var_url + 'var': var_alias }, { 'value': [1] @@ -3457,7 +3491,7 @@ def test_in_expression_list_floats(self): 'function': 'in', 'args': [ { - 'variable': var_url + 'var': var_alias }, { 'value': [1.0] @@ -3507,7 +3541,7 @@ def test_float_conversion_integer_in_list(self): "function": "in", "args": [ { - "variable": "my_var" + "var": "my_var" }, { "value": [ @@ -3523,7 +3557,7 @@ def test_string_no_need_conversion_in_list(self): "function": "in", "args": [ { - "variable": "my_var" + "var": "my_var" }, { "value": [ @@ -3542,7 +3576,7 @@ def test_and(self): 'function': '>', 'args': [ { - 'variable': 'age' + 'var': 'age' }, { 'value': 1 @@ -3553,7 +3587,7 @@ def test_and(self): 'function': '==', 'args': [ { - 'variable': 'favcolor' + 'var': 'favcolor' }, { 'value': 2 @@ -3575,7 +3609,7 @@ def test_nested_or(self): 'function': '>', 'args': [ { - 'variable': 'age' + 'var': 'age' }, { 'value': 1 @@ -3589,7 +3623,7 @@ def test_nested_or(self): 'function': '==', 'args': [ { - 'variable': 'favcolor' + 'var': 'favcolor' }, { 'value': 2 @@ -3600,7 +3634,7 @@ def test_nested_or(self): 'function': '==', 'args': [ { - 'variable': 'genre' + 'var': 'genre' }, { 'value': 1 @@ -3624,7 +3658,7 @@ def test_complex(self): 'function': '>', 'args': [ { - 'variable': 'age' + 'var': 'age' }, { 'value': 55 @@ -3641,7 +3675,7 @@ def test_complex(self): 'function': '==', 'args': [ { - 'variable': 'genre' + 'var': 'genre' }, { 'value': 1 @@ -3652,7 +3686,7 @@ def test_complex(self): 'function': '==', 'args': [ { - 'variable': 'favfruit' + 'var': 'favfruit' }, { 'value': 9 @@ -3665,7 +3699,7 @@ def test_complex(self): 'function': 'in', 'args': [ { - 'variable': 'favcolor' + 'var': 'favcolor' }, { 'value': [3, 4, 5] @@ -3681,12 +3715,12 @@ def test_complex(self): cel = prettify(expr) assert expected == cel - def test_variable_url(self): + def test_variable_alias(self): expr = { 'function': '==', 'args': [ { - 'variable': 'https://host.com/api/datasets/123/variables/001/' + 'var': 'age' }, { 'value': 1 @@ -3704,15 +3738,14 @@ def test_variable_url(self): expected = 'age == 1' cel = prettify(expr, ds) assert expected == cel - ds.resource.session.get.assert_called_with('https://host.com/api/datasets/123/variables/001/') def test_square_bracket_subvariables(self): - subvariable_url = 'https://host.com/api/datasets/123/variables/001/subvariables/abc/' expr = { 'function': '==', 'args': [ { - 'variable': subvariable_url + 'var': 'array_alias', + 'axes': ['sv_1'] }, { 'value': 1 @@ -3734,10 +3767,10 @@ def test_square_bracket_subvariables(self): # Prepare array response2 = mock.MagicMock() - response2.payload.body = {"alias": 'array_variable'} + response2.payload.body = {"alias": 'array_alias'} ds.resource.session.get.side_effect = [response1, response2] - expected = 'array_variable[subvar_1] == 1' + expected = 'array_alias[sv_1] == 1' assert prettify(expr, ds) == expected def test_variable_url_no_dataset(self): @@ -3745,7 +3778,7 @@ def test_variable_url_no_dataset(self): 'function': '==', 'args': [ { - 'variable': 'https://host.com/api/datasets/123/variables/001/' + 'var': 'var_alias' }, { 'value': 1 @@ -3753,13 +3786,8 @@ def test_variable_url_no_dataset(self): ] } - with pytest.raises(Exception) as err: - prettify(expr) + assert prettify(expr) == "var_alias == 1" - assert str(err.value) == ( - 'Valid Dataset instance is required to resolve variable urls ' - 'in the expression' - ) def test_parse_equal_string(self): expr_obj = { @@ -4438,7 +4466,7 @@ def test_iso8601_complete(self): "function": "<", "args": [ { - "variable": "starttime" + "var": "starttime" }, { "value": "2016-12-21T12:00:00+00:00" @@ -4452,7 +4480,7 @@ def test_iso8601_wo_tzinfo(self): "function": "<", "args": [ { - "variable": "starttime" + "var": "starttime" }, { "value": "2016-12-21T12:00:00" @@ -4466,7 +4494,7 @@ def test_iso8601_day_hour_minute_sec(self): "function": "<", "args": [ { - "variable": "starttime" + "var": "starttime" }, { "value": "2016-12-21T12:00:00" @@ -4480,7 +4508,7 @@ def test_iso8601_day_hour_minute(self): "function": "<", "args": [ { - "variable": "starttime" + "var": "starttime" }, { "value": "2016-12-21T12:00" @@ -4494,7 +4522,7 @@ def test_iso8601_day_hour(self): "function": "<", "args": [ { - "variable": "starttime" + "var": "starttime" }, { "value": "2016-12-21T12" @@ -4508,7 +4536,7 @@ def test_iso8601_day(self): "function": "<", "args": [ { - "variable": "starttime" + "var": "starttime" }, { "value": "2016-12-21" @@ -4522,7 +4550,7 @@ def test_iso8601_month(self): "function": "<", "args": [ { - "variable": "starttime" + "var": "starttime" }, { "value": "2016-12" @@ -4536,7 +4564,7 @@ def test_iso8601_year(self): "function": "<", "args": [ { - "variable": "starttime" + "var": "starttime" }, { "value": "2016" @@ -4575,7 +4603,6 @@ def test_process_expression(self): var_id = '0001' var_alias = 'starttime' var_type = 'datetime' - var_url = '%svariables/%s/' % (self.ds_url, var_id) ds = self.mock_dataset(var_id, var_alias, var_type) expr = "starttime < '2016-12-21'" parsed = parse_expr(expr) @@ -4584,7 +4611,7 @@ def test_process_expression(self): 'function': '<', 'args': [ { - 'variable': var_url + 'var': var_alias }, { 'value': '2016-12-21' diff --git a/scrunch/tests/test_recodes.py b/scrunch/tests/test_recodes.py index e1189d99..6a319790 100644 --- a/scrunch/tests/test_recodes.py +++ b/scrunch/tests/test_recodes.py @@ -54,11 +54,11 @@ def mr_in(mr_self, mr_alias, groups, parent_subvariables): RECODES_PAYLOAD = { 'element': 'shoji:entity', 'body': { - 'alias': 'alias', + 'alias': 'derived_variable_alias', 'derivation': { 'function': 'combine_categories', 'args': [ - {'variable': var_url}, + {'var': 'dependency_variable_alias'}, {'value': [ {'combined_ids': [2, 3], 'missing': False, 'id': 1, 'name': 'China'}, {'combined_ids': [1], 'missing': False, 'id': 2, 'name': 'Other'} @@ -151,14 +151,20 @@ def test_combine_categories_from_alias(self): resource = mock.MagicMock() resource.body = {'name': 'mocked_dataset'} entity_mock = mock.MagicMock() - entity_mock.entity.self = var_url + entity_mock.__getitem__.side_effect = lambda key: 'dependency_variable_alias' if key == 'alias' else None resource.variables.by.return_value = { - 'test': entity_mock, + 'dependency_variable_alias': entity_mock, } resource.variables.index = {} ds = MutableDataset(resource) with pytest.raises(ValueError) as err: - ds.combine_categorical('test', CATEGORY_MAP, CATEGORY_NAMES, name='name', alias='alias') + ds.combine_categorical( + 'dependency_variable_alias', + CATEGORY_MAP, + CATEGORY_NAMES, + name='name', + alias='derived_variable_alias' + ) ds.resource.variables.create.assert_called_with(RECODES_PAYLOAD) assert 'Entity mocked_dataset has no (sub)variable' in str(err.value) @@ -168,18 +174,25 @@ def test_combine_categories_from_entity(self): entity_mock = mock.MagicMock() entity_mock.entity.self = var_url resource.variables.by.return_value = { - 'test': entity_mock + 'dependency_variable_alias': entity_mock } resource.variables.index = {} # Var not present # mock a Tuple object tuple_mock = mock.MagicMock() tuple_mock.entity.self = var_url + tuple_mock.__getitem__.side_effect = lambda key: 'dependency_variable_alias' if key == 'alias' else None entity = Variable(tuple_mock, resource) ds = MutableDataset(resource) with pytest.raises(ValueError) as err: - ds.combine_categorical(entity, CATEGORY_MAP, CATEGORY_NAMES, name='name', alias='alias') + ds.combine_categorical( + entity, + CATEGORY_MAP, + CATEGORY_NAMES, + name='name', + alias='derived_variable_alias' + ) ds.resource.variables.create.assert_called_with(RECODES_PAYLOAD) assert 'Entity mocked_dataset has no (sub)variable' in str(err.value) @@ -368,13 +381,13 @@ def test_recode_categoricals(self, get_dataset_mock): }, { 'function': 'in', 'args': [ - {'variable': 'http://test.crunch.io/api/datasets/123/variables/00001/'}, + {'var': 'sexuality'}, {'value': [1]} ] }, { 'function': 'in', 'args': [ - {'variable': 'http://test.crunch.io/api/datasets/123/variables/00001/'}, + {'var': 'sexuality'}, {'value': [2, 3, 4, 5]} ] }] diff --git a/scrunch/variables.py b/scrunch/variables.py index 0c7cbbcf..9ed38086 100644 --- a/scrunch/variables.py +++ b/scrunch/variables.py @@ -56,6 +56,9 @@ def combinations_from_map(map, categories, missing): return combinations +# NOTE: This needs to stay the same because the translation +# to zz9 is done on cr.lib level, and that's where we need +# to convert variables from urls to aliases def combine_responses_expr(variable_url, responses): return { 'function': 'combine_responses', @@ -67,11 +70,11 @@ def combine_responses_expr(variable_url, responses): } -def combine_categories_expr(variable_url, combinations): +def combine_categories_expr(var_alias, combinations): return { 'function': 'combine_categories', 'args': [{ - 'variable': variable_url + 'var': var_alias }, { 'value': combinations }]