diff --git a/examples/BUILD b/examples/BUILD index 1d4cf71..bc577a6 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -120,14 +120,14 @@ jsonnet_to_json_test( jsonnet_library( name = "code_library_lib", srcs = ["code_library.libsonnet"], - deps = [":workflow"] + deps = [":workflow"], ) jsonnet_to_json_test( name = "extvar_code_library_test", size = "small", src = "extvar_code_library.jsonnet", - ext_code_libraries = { ":code_library_lib": "codefile" }, + ext_code_libraries = {":code_library_lib": "codefile"}, golden = "extvar_files_library_golden.json", ) @@ -135,8 +135,8 @@ jsonnet_to_json_test( name = "tla_code_library_test", size = "small", src = "tla_code_library.jsonnet", - tla_code_libraries = { ":code_library_lib": "tla_code" }, golden = "tla_code_library_golden.json", + tla_code_libraries = {":code_library_lib": "tla_code"}, ) jsonnet_to_json_test( @@ -336,3 +336,46 @@ jsonnet_to_json( "multiple_outs_nested_asymmetric/file.json", ], ) + +jsonnet_library( + name = "extvar-lib", + srcs = [ + "extvar.libsonnet", + ], + ext_code_envs = ["MYJSONNET"], + ext_code_libraries = { + ":code_library_lib": "ext_lib", + }, + ext_str_envs = ["MYTEST"], + ext_str_files = { + ":test_str_files": "str_files", + }, + ext_strs = { + "non_stamp": "non_stamp", + "mydefine": "$(mydefine)", + "k8s": "{STABLE_K8S_CLUSTER}", + }, +) + +jsonnet_to_json_test( + name = "extvar-lib-test", + src = "extvar-library.jsonnet", + golden = "extvar-library.json", + deps = [ + ":extvar-lib", + ], +) + +jsonnet_to_json( + name = "extvar-lib-json", + src = "extvar-library.jsonnet", + outs = ["ext-lib.json"], + ext_code_envs = ["MYJSONNET"], + ext_str_envs = ["MYTEST"], + ext_strs = { + "non_stamp": "non_stamp", + }, + deps = [ + ":extvar-lib", + ], +) diff --git a/examples/extvar-library.json b/examples/extvar-library.json new file mode 100644 index 0000000..8e92b0e --- /dev/null +++ b/examples/extvar-library.json @@ -0,0 +1,13 @@ +{ + "code_env": { + "code": "some code" + }, + "library": { + "workflow": { } + }, + "k8s": "{STABLE_K8S_CLUSTER}", + "mydefine": "", + "non_stamp": "non_stamp", + "str_env": "test", + "str_files": "this is great\n" +} diff --git a/examples/extvar-library.jsonnet b/examples/extvar-library.jsonnet new file mode 100644 index 0000000..e49ba66 --- /dev/null +++ b/examples/extvar-library.jsonnet @@ -0,0 +1 @@ +import 'extvar.libsonnet' diff --git a/examples/extvar.libsonnet b/examples/extvar.libsonnet new file mode 100644 index 0000000..e02eb5a --- /dev/null +++ b/examples/extvar.libsonnet @@ -0,0 +1,9 @@ +{ + non_stamp: std.extVar('non_stamp'), + mydefine: std.extVar('mydefine'), + k8s: std.extVar('k8s'), + str_files: std.extVar('str_files'), + code_env: std.extVar('MYJSONNET'), + str_env: std.extVar('MYTEST'), + library: std.extVar('ext_lib'), +} diff --git a/jsonnet/jsonnet.bzl b/jsonnet/jsonnet.bzl index 5b8c632..09ee8e0 100644 --- a/jsonnet/jsonnet.bzl +++ b/jsonnet/jsonnet.bzl @@ -25,6 +25,7 @@ JsonnetLibraryInfo = provider( "imports": "Depset of Strings containing import flags set by transitive dependency targets.", "short_imports": "Depset of Strings containing import flags set by transitive dependency targets, when invoking Jsonnet as part of a test where dependencies are stored in runfiles.", "transitive_jsonnet_files": "Depset of Files containing sources of transitive dependencies", + "transitive_extvars": "Dict of extvar from transitive dependencies", }, ) @@ -51,13 +52,14 @@ def _get_import_paths(label, files, imports, short_path): for im in imports ] -def _setup_deps(deps, tla_code_libraries = {}, ext_code_libraries = {}): +def _setup_deps(deps, tla_code_libraries = {}, ext_code_libraries = {}, transitive_extvars = {}): """Collects source files and import flags of transitive dependencies. Args: deps: List of deps labels from ctx.attr.deps. tla_code_libraries: Dict of labels to names from ctx.attr.tla_code_files. ext_code_libraries: List of deps labels from ctx.attr.ext_code_files. + transitive_extvars: Dict of extvar to values build from _make_extvar_dict Returns: Returns a struct containing the following fields: @@ -68,6 +70,9 @@ def _setup_deps(deps, tla_code_libraries = {}, ext_code_libraries = {}): short_imports: Depset of Strings containing import flags set by transitive dependency targets, when invoking Jsonnet as part of a test where dependencies are stored in runfiles. + transitive_extvars: Dict of extvar to values that has merged the + input value with all extvars of its depdencies. + """ transitive_sources = [] imports = [] @@ -76,22 +81,209 @@ def _setup_deps(deps, tla_code_libraries = {}, ext_code_libraries = {}): transitive_sources.append(dep[JsonnetLibraryInfo].transitive_jsonnet_files) imports.append(dep[JsonnetLibraryInfo].imports) short_imports.append(dep[JsonnetLibraryInfo].short_imports) + transitive_extvars = _merge_extvars(transitive_extvars, dep[JsonnetLibraryInfo].transitive_extvars) for code_file in tla_code_libraries.keys() + ext_code_libraries.keys(): transitive_sources.append(code_file[JsonnetLibraryInfo].transitive_jsonnet_files) imports.append(code_file[JsonnetLibraryInfo].imports) short_imports.append(code_file[JsonnetLibraryInfo].short_imports) + transitive_extvars = _merge_extvars(transitive_extvars, code_file[JsonnetLibraryInfo].transitive_extvars) return struct( imports = depset(transitive = imports), short_imports = depset(transitive = short_imports), transitive_sources = depset(transitive = transitive_sources, order = "postorder"), + transitive_extvars = transitive_extvars, ) +def _make_extvar_dict( + label, + ext_code, + ext_code_envs, + ext_code_files, + ext_code_libraries, + ext_str_envs, + ext_str_files, + ext_strs): + """Transforms input ext_* attributes and builds a transitive_extvars dict + + Args: + label: Label to track source of ext var for making debugging messages + ext_code: Dict of variable names to code, maps from ctx.attr.ext_code + ext_code_envs: List of variable names that map to environment variables, maps from ctx.attr.ext_code_envs + ext_code_files: Dict of Label or File to variable names, maps from ctx.attr.ext_code_files + ext_code_libraries: Dict of Label to variable names, maps from ctx.attr.ext_code_libraries + ext_str_envs: List of variable names to strings from environment, maps from ctx.attr.ext_str_envs + ext_str_files: Dict of Label or File to variable names, maps from ctx.attr.ext_str_files + ext_strs: Dict of variable names to strings, maps from ctx.attr.ext_strs + + Returns: + Dictionary with keys are variable names, and values a dict containing + type: The type of extvar it will be in jsonnet: string or code + value: The string, code, or File depending on type + sources: List of labels that define the extvar + """ + extvars = dict() + label = str(label) + + # extvar_lists is a list of tuple (extvar: str, value: None | str | File | JsonnetInfo, extvar_type: str) + # The `None` value are used by environment + # Collect all the Code extvars + # ext_code, dict[extvar, str_value] + extvar_code_lists = zip(ext_code.keys(), ext_code.values()) + + # ext_code_envs, list[extvar] + extvar_code_lists.extend(zip(ext_code_envs, [None] * len(ext_code_envs))) + + # ext_code_files, dict[label, extvar] + extvar_code_lists.extend(zip(ext_code_files.values(), ext_code_files.keys())) + + # ext_code_libraries, dict[label, extvar] + extvar_code_lists.extend(zip(ext_code_libraries.values(), ext_code_libraries.keys())) + + for key, val in extvar_code_lists: + _make_extvar_dict_update(extvars, "code", key, val, label) + + # Collect all of the String extvars + # ext_str_envs, list[extvar] + extvar_str_lists = zip(ext_str_envs, [None] * len(ext_str_envs)) + + # ext_str_files, dict[label, extvar] + extvar_str_lists.extend(zip(ext_str_files.values(), ext_str_files.keys())) + + # ext_strs, dict[extvar, str] + extvar_str_lists.extend(zip(ext_strs.keys(), ext_strs.values())) + + for key, val in extvar_str_lists: + _make_extvar_dict_update(extvars, "string", key, val, label) + + return extvars + +def _make_extvar_dict_update(extvars, extvar_type, extvar_name, value, label): + """Adds an entry to a given extrvars dict and validates its uniqueness + + Args: + extvars: Dict of extvars to be added to + extvar_type: String of either "string" or "code" + extvar_name: String of the extvar variable name + value: Either a None, string, File or Target + label: String of the package this extvar is defined in + + Returns: + None, modifies the given extvars input in-place + """ + if extvar_name in extvars: + fail("duplicate extvar '{}' of type {} and {}" + .format(extvar_name, extvar_type, extvars[extvar_name]["type"])) + + if type(value) == "string" or type(value) == "File" or value == None: + pass + elif type(value) == "Target": + value = value[DefaultInfo].files.to_list()[0] + else: + fail("unknown type of value {} for {} in {}".format(type(value), extvar_name, label)) + + extvars.update([[extvar_name, { + "value": value, + "type": extvar_type, + "sources": [label], + }]]) + +def _extvar_to_arguments(transitive_extvars, short_path = False): + """Converts an transitive_extvars to command line arguments + + Args: + transitive_extvars: dict of extvar from _make_extvar_dict + short_path: Boolean - if the short_path of files should be used + + Returns: + List of strings of arguments for extvars + """ + args = [] + for key, val in transitive_extvars.items(): + # The --ext-str-* and --ext-code-* flag families are interchangable, + # so the `type` is used to determine which to use. + flag_type = "str" if val["type"] == "string" else val["type"] + + # Each different type of value is formatted in the flags differently + if val["value"] == None: + # Environment flags + args.append("--ext-%s %s" % (flag_type, _quote(key))) + elif type(val["value"]) == "string": + # String flags + args.append("--ext-%s %s=%s" % (flag_type, _quote(key), _quote(val["value"]))) + elif type(val["value"]) == "File": + # Files and library flags + file = val["value"] + file_path = file.short_path if short_path else file.path + args.append("--ext-%s-file %s=%s" % (flag_type, _quote(key), _quote(file_path))) + else: + fail("The {} key has an unknown extvar type {}: {}".format(key, type(val["value"]), val["sources"])) + + return args + +def _merge_extvars(left, right): + """Merges two extvar dicts together + + In the case of duplicates extvar (keys of the dict): + 1. If type & value match: the inner sources are merged + 2. If type or value mismatches: raises an error + + Args: + left: A dictonary made from _make_ext_dict + right: A dictonary made from _make_ext_dict + + Returns: + A _make_ext_dict compatible dict + """ + result = dict(left) + + for (var, right_val) in right.items(): + # Check if the variable name has been used already + if var in left: + left_val = left[var] + + # Check if is the same type & value + if left_val["type"] != right_val["type"]: + # If the types are different + fail("extvar {} is defined in multiple places with different types: {}" + .format(var, left_val["sources"] + right_val["sources"])) + elif left_val["value"] != right_val["value"]: + fail("extvar {} is defined in multiple places with different values: {}" + .format(var, left_val["sources"] + right_val["sources"])) + else: + # type & value match! + # Collect the sources to provide better error messages if there ever is a mismatch + result[var]["sources"].extend(right_val["sources"]) + else: + # Simple case, right side has a new variable + result[var] = right_val + + return result + def _jsonnet_library_impl(ctx): """Implementation of the jsonnet_library rule.""" - depinfo = _setup_deps(ctx.attr.deps) - sources = depset(ctx.files.srcs, transitive = [depinfo.transitive_sources]) + transitive_extvars = _make_extvar_dict( + ctx.label, + ctx.attr.ext_code, + ctx.attr.ext_code_envs, + ctx.attr.ext_code_files, + ctx.attr.ext_code_libraries, + ctx.attr.ext_str_envs, + ctx.attr.ext_str_files, + ctx.attr.ext_strs, + ) + + depinfo = _setup_deps( + ctx.attr.deps, + ext_code_libraries = ctx.attr.ext_code_libraries, + transitive_extvars = transitive_extvars, + ) + + sources = depset( + ctx.files.srcs + ctx.files.ext_code_files + ctx.files.ext_str_files, + transitive = [depinfo.transitive_sources], + ) imports = depset( _get_import_paths(ctx.label, ctx.files.srcs, ctx.attr.imports, False), transitive = [depinfo.imports], @@ -116,6 +308,7 @@ def _jsonnet_library_impl(ctx): imports = imports, short_imports = short_imports, transitive_jsonnet_files = sources, + transitive_extvars = depinfo.transitive_extvars, ), ] @@ -193,14 +386,30 @@ def _jsonnet_to_json_impl(ctx): jsonnet_tla_code_files = ctx.attr.tla_code_files jsonnet_tla_code_libraries = ctx.attr.tla_code_libraries - depinfo = _setup_deps(ctx.attr.deps, jsonnet_tla_code_libraries, jsonnet_ext_code_libraries) - jsonnet_ext_strs, strs_stamp_inputs = _make_stamp_resolve(ctx.attr.ext_strs, ctx, False) jsonnet_ext_code, code_stamp_inputs = _make_stamp_resolve(ctx.attr.ext_code, ctx, False) jsonnet_tla_strs, tla_strs_stamp_inputs = _make_stamp_resolve(ctx.attr.tla_strs, ctx, False) jsonnet_tla_code, tla_code_stamp_inputs = _make_stamp_resolve(ctx.attr.tla_code, ctx, False) stamp_inputs = strs_stamp_inputs + code_stamp_inputs + tla_strs_stamp_inputs + tla_code_stamp_inputs + transitive_extvars = _make_extvar_dict( + ctx.label, + jsonnet_ext_code, + jsonnet_ext_code_envs, + dict(zip(jsonnet_ext_code_files, jsonnet_ext_code_file_vars)), + jsonnet_ext_code_libraries, + jsonnet_ext_str_envs, + dict(zip(jsonnet_ext_str_files, jsonnet_ext_str_file_vars)), + jsonnet_ext_strs, + ) + + depinfo = _setup_deps( + ctx.attr.deps, + jsonnet_tla_code_libraries, + jsonnet_ext_code_libraries, + transitive_extvars, + ) + if len(jsonnet_ext_str_file_vars) != len(jsonnet_ext_str_files): fail("Mismatch of ext_str_file_vars ({}) to ext_str_files ({})".format(jsonnet_ext_str_file_vars, jsonnet_ext_str_files)) @@ -220,20 +429,7 @@ def _jsonnet_to_json_impl(ctx): ["-J " + shell.quote(im) for im in _get_import_paths(ctx.label, [ctx.file.src], ctx.attr.imports, False)] + ["-J " + shell.quote(im) for im in depinfo.imports.to_list()] + other_args + - ["--ext-str %s=%s" % - (_quote(key), _quote(val)) for key, val in jsonnet_ext_strs.items()] + - ["--ext-str '%s'" % - ext_str_env for ext_str_env in jsonnet_ext_str_envs] + - ["--ext-code %s=%s" % - (_quote(key), _quote(val)) for key, val in jsonnet_ext_code.items()] + - ["--ext-code %s" % - ext_code_env for ext_code_env in jsonnet_ext_code_envs] + - ["--ext-str-file %s=%s" % - (var, jfile.path) for var, jfile in zip(jsonnet_ext_str_file_vars, jsonnet_ext_str_files)] + - ["--ext-code-file %s=%s" % - (var, jfile.path) for var, jfile in zip(jsonnet_ext_code_file_vars, jsonnet_ext_code_files)] + - ["--ext-code-file %s=%s" % - (_quote(val), _quote(key[DefaultInfo].files.to_list()[0].path)) for key, val in jsonnet_ext_code_libraries.items()] + + _extvar_to_arguments(depinfo.transitive_extvars) + ["--tla-str %s=%s" % (_quote(key), _quote(val)) for key, val in jsonnet_tla_strs.items()] + ["--tla-str '%s'" % @@ -386,7 +582,6 @@ fi def _jsonnet_to_json_test_impl(ctx): """Implementation of the jsonnet_to_json_test rule.""" - depinfo = _setup_deps(ctx.attr.deps, ctx.attr.tla_code_libraries, ctx.attr.ext_code_libraries) golden_files = [] diff_command = "" @@ -423,6 +618,7 @@ def _jsonnet_to_json_test_impl(ctx): jsonnet_ext_code_files = ctx.files.ext_code_files jsonnet_ext_code_file_vars = ctx.attr.ext_code_file_vars jsonnet_ext_code_libraries = ctx.attr.ext_code_libraries + jsonnet_tla_str_envs = ctx.attr.tla_str_envs jsonnet_tla_code_envs = ctx.attr.tla_code_envs jsonnet_tla_str_files = ctx.attr.tla_str_files @@ -435,6 +631,24 @@ def _jsonnet_to_json_test_impl(ctx): jsonnet_tla_code, tla_code_stamp_inputs = _make_stamp_resolve(ctx.attr.tla_code, ctx, True) stamp_inputs = strs_stamp_inputs + code_stamp_inputs + tla_strs_stamp_inputs + tla_code_stamp_inputs + transitive_extvars = _make_extvar_dict( + ctx.label, + jsonnet_ext_code, + jsonnet_ext_code_envs, + dict(zip(jsonnet_ext_code_files, jsonnet_ext_code_file_vars)), + jsonnet_ext_code_libraries, + jsonnet_ext_str_envs, + dict(zip(jsonnet_ext_str_files, jsonnet_ext_str_file_vars)), + jsonnet_ext_strs, + ) + + depinfo = _setup_deps( + ctx.attr.deps, + jsonnet_tla_code_libraries, + jsonnet_ext_code_libraries, + transitive_extvars, + ) + if len(jsonnet_ext_str_file_vars) != len(jsonnet_ext_str_files): fail("Mismatch of ext_str_file_vars ({}) to ext_str_files ({})".format(jsonnet_ext_str_file_vars, jsonnet_ext_str_files)) @@ -447,20 +661,7 @@ def _jsonnet_to_json_test_impl(ctx): ["-J " + shell.quote(im) for im in _get_import_paths(ctx.label, [ctx.file.src], ctx.attr.imports, True)] + ["-J " + shell.quote(im) for im in depinfo.short_imports.to_list()] + other_args + - ["--ext-str %s=%s" % - (_quote(key), _quote(val)) for key, val in jsonnet_ext_strs.items()] + - ["--ext-str %s" % - ext_str_env for ext_str_env in jsonnet_ext_str_envs] + - ["--ext-code %s=%s" % - (_quote(key), _quote(val)) for key, val in jsonnet_ext_code.items()] + - ["--ext-code %s" % - ext_code_env for ext_code_env in jsonnet_ext_code_envs] + - ["--ext-str-file %s=%s" % - (var, jfile.short_path) for var, jfile in zip(jsonnet_ext_str_file_vars, jsonnet_ext_str_files)] + - ["--ext-code-file %s=%s" % - (var, jfile.short_path) for var, jfile in zip(jsonnet_ext_code_file_vars, jsonnet_ext_code_files)] + - ["--ext-code-file %s=%s" % - (_quote(val), _quote(key[DefaultInfo].files.to_list()[0].short_path)) for key, val in jsonnet_ext_code_libraries.items()] + + _extvar_to_arguments(depinfo.transitive_extvars, short_path = True) + ["--tla-str %s=%s" % (_quote(key), _quote(val)) for key, val in jsonnet_tla_strs.items()] + ["--tla-str '%s'" % @@ -545,6 +746,30 @@ _jsonnet_library_attrs = { doc = "List of `.jsonnet` files that comprises this Jsonnet library", allow_files = _JSONNET_FILETYPE, ), + "ext_code": attr.string_dict( + doc = "Include code from the dict value via extvar. Variable name matches the key", + ), + "ext_code_envs": attr.string_list( + doc = "Include code from an environment variable via extvar. Variable name matches the environment variable name", + ), + "ext_code_files": attr.label_keyed_string_dict( + doc = "Include code from a file from dict key via extvar. Variable name matches the value", + allow_files = True, + ), + "ext_code_libraries": attr.label_keyed_string_dict( + doc = "Include jsonnet_library as an extvar with the key value", + providers = [JsonnetLibraryInfo], + ), + "ext_str_envs": attr.string_list( + doc = "Include string from an environment variable via extvar. Variable name matches the environment variable name", + ), + "ext_str_files": attr.label_keyed_string_dict( + doc = "Include string from a file from dict key via extvar. Variable name matches the value", + allow_files = True, + ), + "ext_strs": attr.string_dict( + doc = "Include string from the dict value via extvar. Variable name matches the key", + ), } jsonnet_library = rule(