From aa76f5203d69722e620d40eb2aced7bcaf8b7a3a Mon Sep 17 00:00:00 2001 From: David Barnett Date: Sun, 14 Sep 2025 20:48:29 +1000 Subject: [PATCH 1/5] add support for extvars for jsonnet_library --- examples/BUILD | 43 ++++++ examples/extvar-library.json | 13 ++ examples/extvar-library.jsonnet | 1 + examples/extvar.libsonnet | 9 ++ jsonnet/jsonnet.bzl | 239 +++++++++++++++++++++++++++----- 5 files changed, 272 insertions(+), 33 deletions(-) create mode 100644 examples/extvar-library.json create mode 100644 examples/extvar-library.jsonnet create mode 100644 examples/extvar.libsonnet diff --git a/examples/BUILD b/examples/BUILD index 1d4cf71..d79a586 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -336,3 +336,46 @@ jsonnet_to_json( "multiple_outs_nested_asymmetric/file.json", ], ) + +jsonnet_library( + name = "extvar-lib", + srcs = [ + "extvar.libsonnet", + ], + ext_strs = { + "non_stamp": "non_stamp", + "mydefine": "$(mydefine)", + "k8s": "{STABLE_K8S_CLUSTER}", + }, + ext_str_files = { + ":test_str_files": "str_files" + }, + ext_code_envs = ["MYJSONNET"], + ext_str_envs = ["MYTEST"], + ext_code_libraries = { + ":code_library_lib": "ext_lib" + }, +) + +jsonnet_to_json_test( + name = "extvar-lib-test", + src = "extvar-library.jsonnet", + golden = "extvar-library.json", + deps = [ + ":extvar-lib", + ], +) + +jsonnet_to_json( + name = "extvar-lib-json", + outs = ["ext-lib.json"], + src = "extvar-library.jsonnet", + deps = [ + ":extvar-lib", + ], + ext_strs = { + "non_stamp": "non_stamp", + }, + ext_code_envs = ["MYJSONNET"], + ext_str_envs = ["MYTEST"], +) diff --git a/examples/extvar-library.json b/examples/extvar-library.json new file mode 100644 index 0000000..8e92b0e --- /dev/null +++ b/examples/extvar-library.json @@ -0,0 +1,13 @@ +{ + "code_env": { + "code": "some code" + }, + "library": { + "workflow": { } + }, + "k8s": "{STABLE_K8S_CLUSTER}", + "mydefine": "", + "non_stamp": "non_stamp", + "str_env": "test", + "str_files": "this is great\n" +} diff --git a/examples/extvar-library.jsonnet b/examples/extvar-library.jsonnet new file mode 100644 index 0000000..e49ba66 --- /dev/null +++ b/examples/extvar-library.jsonnet @@ -0,0 +1 @@ +import 'extvar.libsonnet' diff --git a/examples/extvar.libsonnet b/examples/extvar.libsonnet new file mode 100644 index 0000000..e02eb5a --- /dev/null +++ b/examples/extvar.libsonnet @@ -0,0 +1,9 @@ +{ + non_stamp: std.extVar('non_stamp'), + mydefine: std.extVar('mydefine'), + k8s: std.extVar('k8s'), + str_files: std.extVar('str_files'), + code_env: std.extVar('MYJSONNET'), + str_env: std.extVar('MYTEST'), + library: std.extVar('ext_lib'), +} diff --git a/jsonnet/jsonnet.bzl b/jsonnet/jsonnet.bzl index 5b8c632..21555b7 100644 --- a/jsonnet/jsonnet.bzl +++ b/jsonnet/jsonnet.bzl @@ -25,6 +25,7 @@ JsonnetLibraryInfo = provider( "imports": "Depset of Strings containing import flags set by transitive dependency targets.", "short_imports": "Depset of Strings containing import flags set by transitive dependency targets, when invoking Jsonnet as part of a test where dependencies are stored in runfiles.", "transitive_jsonnet_files": "Depset of Files containing sources of transitive dependencies", + "transitive_extvars": "Dict of extvar from transitive dependencies", }, ) @@ -51,7 +52,7 @@ def _get_import_paths(label, files, imports, short_path): for im in imports ] -def _setup_deps(deps, tla_code_libraries = {}, ext_code_libraries = {}): +def _setup_deps(deps, tla_code_libraries = {}, ext_code_libraries = {}, transitive_extvars = {}): """Collects source files and import flags of transitive dependencies. Args: @@ -76,22 +77,171 @@ def _setup_deps(deps, tla_code_libraries = {}, ext_code_libraries = {}): transitive_sources.append(dep[JsonnetLibraryInfo].transitive_jsonnet_files) imports.append(dep[JsonnetLibraryInfo].imports) short_imports.append(dep[JsonnetLibraryInfo].short_imports) + transitive_extvars = _merge_extvars(transitive_extvars, dep[JsonnetLibraryInfo].transitive_extvars) for code_file in tla_code_libraries.keys() + ext_code_libraries.keys(): transitive_sources.append(code_file[JsonnetLibraryInfo].transitive_jsonnet_files) imports.append(code_file[JsonnetLibraryInfo].imports) short_imports.append(code_file[JsonnetLibraryInfo].short_imports) + transitive_extvars = _merge_extvars(transitive_extvars, code_file[JsonnetLibraryInfo].transitive_extvars) return struct( imports = depset(transitive = imports), short_imports = depset(transitive = short_imports), transitive_sources = depset(transitive = transitive_sources, order = "postorder"), + transitive_extvars = transitive_extvars, ) +def _make_extvar_dict(label, + ext_code, + ext_code_envs, + ext_code_files, + ext_code_libraries, + ext_str_envs, + ext_str_files, + ext_strs, + ): + extvars = dict() + for key, code in ext_code.items(): + if key in extvars: + fail("duplicate extVar '{}': {}".format(key, extvars.keys())) + extvars[key] = { + 'value': code, + 'type': 'code', + 'sources': [label], + } + for key in ext_code_envs: + if key in extvars: + fail("duplicate extVar '{}': {}".format(key, extvars.keys())) + extvars[key] = { + 'value': '', + 'type': 'code_env', + 'sources': [label], + } + for file, key in ext_code_files.items(): + if key in extvars: + fail("duplicate extVar '{}': {}".format(key, extvars.keys())) + + files = [file] + if type(file) != "File": + files = file[DefaultInfo].files.to_list() + + extvars[key] = { + 'value': files, + 'type': 'code_file', + 'sources': [label], + } + for file, key in ext_code_libraries.items(): + if key in extvars: + fail("duplicate extVar '{}': {}".format(key, extvars.keys())) + extvars[key] = { + 'value': file[DefaultInfo].files.to_list(), + 'type': 'code_library', + 'sources': [label], + } + for key in ext_str_envs: + if key in extvars: + fail("duplicate extVar '{}': {}".format(key, extvars.keys())) + extvars[key] = { + 'value': '', + 'type': 'string_env', + 'sources': [label], + } + for file, key in ext_str_files.items(): + if key in extvars: + fail("duplicate extVar '{}': {}".format(key, extvars.keys())) + + files = [file] + if type(file) != "File": + files = file[DefaultInfo].files.to_list() + + extvars[key] = { + 'value': files, + 'type': 'string_file', + 'sources': [label], + } + for key, val in ext_strs.items(): + if key in extvars: + fail("duplicate extVar '{}': {}".format(key, extvars.keys())) + extvars[key] = { + 'value': val, + 'type': 'string', + 'sources': [label], + } + return extvars + +def _extvar_to_arguments(transitive_extvars, short_path=False): + args = [] + for key, val in transitive_extvars.items(): + if val['type'] == 'string': + args.append("--ext-str %s=%s" % (_quote(key), _quote(val['value']))) + elif val['type'] == 'string_env': + args.append("--ext-str %s" % _quote(key)) + elif val['type'] == 'string_file': + file = val['value'][0] + args.append("--ext-str-file %s=%s" % (_quote(key), _quote(file.short_path if short_path else file.path))) + elif val['type'] == 'code': + args.append("--ext-code %s=%s" % (_quote(key), _quote(val['value']))) + elif val['type'] == 'code_env': + args.append("--ext-code %s" % _quote(key)) + elif val['type'] == 'code_library' or val['type'] == 'code_file': + file = val['value'][0] + args.append("--ext-code-file %s=%s" % (_quote(key), _quote(file.short_path if short_path else file.path))) + else: + fail("The {} key has an unknown extvar type {}: {}".format(key, val["type"], val["sources"])) + + return args + +def _merge_extvars(left, right): + """Merges two extvar dicts together + + """ + result = dict(left) + + for (var, right_val) in right.items(): + # Check if the variable name has been used already + if var in left: + left_val = left[var] + # Check if is the same type & value + if left_val['type'] != right_val['type']: + # If the types are different + fail("extvar {} is defined in multiple places with different types: {}" + .format(var, left_val['sources'] + right_val['sources'])) + elif left_val['value'] != right_val['value']: + fail("extvar {} is defined in multiple places with different values: {}" + .format(var, left_val['sources'] + right_val['sources'])) + else: + # type & value match! + # Collect the sources to provide better error messages + result[var]['sources'].extend(right_val['sources']) + else: + result[var] = right_val + + return result + def _jsonnet_library_impl(ctx): """Implementation of the jsonnet_library rule.""" - depinfo = _setup_deps(ctx.attr.deps) - sources = depset(ctx.files.srcs, transitive = [depinfo.transitive_sources]) + transitive_extvars = _make_extvar_dict( + ctx.label, + ctx.attr.ext_code, + ctx.attr.ext_code_envs, + ctx.attr.ext_code_files, + ctx.attr.ext_code_libraries, + ctx.attr.ext_str_envs, + ctx.attr.ext_str_files, + ctx.attr.ext_strs, + ) + + depinfo = _setup_deps( + ctx.attr.deps, + ext_code_libraries=ctx.attr.ext_code_libraries, + transitive_extvars=transitive_extvars + ) + + sources = depset( + ctx.files.srcs + ctx.files.ext_code_files + ctx.files.ext_str_files, + transitive = [ depinfo.transitive_sources ] + ) imports = depset( _get_import_paths(ctx.label, ctx.files.srcs, ctx.attr.imports, False), transitive = [depinfo.imports], @@ -116,6 +266,7 @@ def _jsonnet_library_impl(ctx): imports = imports, short_imports = short_imports, transitive_jsonnet_files = sources, + transitive_extvars = transitive_extvars, ), ] @@ -193,7 +344,23 @@ def _jsonnet_to_json_impl(ctx): jsonnet_tla_code_files = ctx.attr.tla_code_files jsonnet_tla_code_libraries = ctx.attr.tla_code_libraries - depinfo = _setup_deps(ctx.attr.deps, jsonnet_tla_code_libraries, jsonnet_ext_code_libraries) + transitive_extvars = _make_extvar_dict( + ctx.label, + jsonnet_ext_code, + jsonnet_ext_code_envs, + dict(zip(jsonnet_ext_code_files, jsonnet_ext_code_file_vars)), + jsonnet_ext_code_libraries, + jsonnet_ext_str_envs, + dict(zip(jsonnet_ext_str_files, jsonnet_ext_str_file_vars)), + jsonnet_ext_strs, + ) + + depinfo = _setup_deps( + ctx.attr.deps, + jsonnet_tla_code_libraries, + jsonnet_ext_code_libraries, + transitive_extvars, + ) jsonnet_ext_strs, strs_stamp_inputs = _make_stamp_resolve(ctx.attr.ext_strs, ctx, False) jsonnet_ext_code, code_stamp_inputs = _make_stamp_resolve(ctx.attr.ext_code, ctx, False) @@ -220,20 +387,7 @@ def _jsonnet_to_json_impl(ctx): ["-J " + shell.quote(im) for im in _get_import_paths(ctx.label, [ctx.file.src], ctx.attr.imports, False)] + ["-J " + shell.quote(im) for im in depinfo.imports.to_list()] + other_args + - ["--ext-str %s=%s" % - (_quote(key), _quote(val)) for key, val in jsonnet_ext_strs.items()] + - ["--ext-str '%s'" % - ext_str_env for ext_str_env in jsonnet_ext_str_envs] + - ["--ext-code %s=%s" % - (_quote(key), _quote(val)) for key, val in jsonnet_ext_code.items()] + - ["--ext-code %s" % - ext_code_env for ext_code_env in jsonnet_ext_code_envs] + - ["--ext-str-file %s=%s" % - (var, jfile.path) for var, jfile in zip(jsonnet_ext_str_file_vars, jsonnet_ext_str_files)] + - ["--ext-code-file %s=%s" % - (var, jfile.path) for var, jfile in zip(jsonnet_ext_code_file_vars, jsonnet_ext_code_files)] + - ["--ext-code-file %s=%s" % - (_quote(val), _quote(key[DefaultInfo].files.to_list()[0].path)) for key, val in jsonnet_ext_code_libraries.items()] + + _extvar_to_arguments(depinfo.transitive_extvars) + ["--tla-str %s=%s" % (_quote(key), _quote(val)) for key, val in jsonnet_tla_strs.items()] + ["--tla-str '%s'" % @@ -386,7 +540,6 @@ fi def _jsonnet_to_json_test_impl(ctx): """Implementation of the jsonnet_to_json_test rule.""" - depinfo = _setup_deps(ctx.attr.deps, ctx.attr.tla_code_libraries, ctx.attr.ext_code_libraries) golden_files = [] diff_command = "" @@ -423,6 +576,7 @@ def _jsonnet_to_json_test_impl(ctx): jsonnet_ext_code_files = ctx.files.ext_code_files jsonnet_ext_code_file_vars = ctx.attr.ext_code_file_vars jsonnet_ext_code_libraries = ctx.attr.ext_code_libraries + jsonnet_tla_str_envs = ctx.attr.tla_str_envs jsonnet_tla_code_envs = ctx.attr.tla_code_envs jsonnet_tla_str_files = ctx.attr.tla_str_files @@ -435,6 +589,24 @@ def _jsonnet_to_json_test_impl(ctx): jsonnet_tla_code, tla_code_stamp_inputs = _make_stamp_resolve(ctx.attr.tla_code, ctx, True) stamp_inputs = strs_stamp_inputs + code_stamp_inputs + tla_strs_stamp_inputs + tla_code_stamp_inputs + transitive_extvars = _make_extvar_dict( + ctx.label, + jsonnet_ext_code, + jsonnet_ext_code_envs, + dict(zip(jsonnet_ext_code_files, jsonnet_ext_code_file_vars)), + jsonnet_ext_code_libraries, + jsonnet_ext_str_envs, + dict(zip(jsonnet_ext_str_files, jsonnet_ext_str_file_vars)), + jsonnet_ext_strs, + ) + + depinfo = _setup_deps( + ctx.attr.deps, + jsonnet_tla_code_libraries, + jsonnet_ext_code_libraries, + transitive_extvars, + ) + if len(jsonnet_ext_str_file_vars) != len(jsonnet_ext_str_files): fail("Mismatch of ext_str_file_vars ({}) to ext_str_files ({})".format(jsonnet_ext_str_file_vars, jsonnet_ext_str_files)) @@ -447,20 +619,7 @@ def _jsonnet_to_json_test_impl(ctx): ["-J " + shell.quote(im) for im in _get_import_paths(ctx.label, [ctx.file.src], ctx.attr.imports, True)] + ["-J " + shell.quote(im) for im in depinfo.short_imports.to_list()] + other_args + - ["--ext-str %s=%s" % - (_quote(key), _quote(val)) for key, val in jsonnet_ext_strs.items()] + - ["--ext-str %s" % - ext_str_env for ext_str_env in jsonnet_ext_str_envs] + - ["--ext-code %s=%s" % - (_quote(key), _quote(val)) for key, val in jsonnet_ext_code.items()] + - ["--ext-code %s" % - ext_code_env for ext_code_env in jsonnet_ext_code_envs] + - ["--ext-str-file %s=%s" % - (var, jfile.short_path) for var, jfile in zip(jsonnet_ext_str_file_vars, jsonnet_ext_str_files)] + - ["--ext-code-file %s=%s" % - (var, jfile.short_path) for var, jfile in zip(jsonnet_ext_code_file_vars, jsonnet_ext_code_files)] + - ["--ext-code-file %s=%s" % - (_quote(val), _quote(key[DefaultInfo].files.to_list()[0].short_path)) for key, val in jsonnet_ext_code_libraries.items()] + + _extvar_to_arguments(depinfo.transitive_extvars, short_path = True) + ["--tla-str %s=%s" % (_quote(key), _quote(val)) for key, val in jsonnet_tla_strs.items()] + ["--tla-str '%s'" % @@ -545,6 +704,20 @@ _jsonnet_library_attrs = { doc = "List of `.jsonnet` files that comprises this Jsonnet library", allow_files = _JSONNET_FILETYPE, ), + "ext_code": attr.string_dict(), + "ext_code_envs": attr.string_list(), + "ext_code_files": attr.label_keyed_string_dict( + allow_files = True, + ), + "ext_code_libraries": attr.label_keyed_string_dict( + doc = "Include jsonnet_library as an extvar with the key value", + providers = [JsonnetLibraryInfo], + ), + "ext_str_envs": attr.string_list(), + "ext_str_files": attr.label_keyed_string_dict( + allow_files = True, + ), + "ext_strs": attr.string_dict(), } jsonnet_library = rule( From d947011bf22f688a3b51e40f1becf339954589b8 Mon Sep 17 00:00:00 2001 From: David Barnett Date: Sun, 14 Sep 2025 21:23:51 +1000 Subject: [PATCH 2/5] docs and buildifier runs --- examples/BUILD | 34 +++++----- jsonnet/jsonnet.bzl | 160 +++++++++++++++++++++++++++++--------------- 2 files changed, 122 insertions(+), 72 deletions(-) diff --git a/examples/BUILD b/examples/BUILD index d79a586..bc577a6 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -120,14 +120,14 @@ jsonnet_to_json_test( jsonnet_library( name = "code_library_lib", srcs = ["code_library.libsonnet"], - deps = [":workflow"] + deps = [":workflow"], ) jsonnet_to_json_test( name = "extvar_code_library_test", size = "small", src = "extvar_code_library.jsonnet", - ext_code_libraries = { ":code_library_lib": "codefile" }, + ext_code_libraries = {":code_library_lib": "codefile"}, golden = "extvar_files_library_golden.json", ) @@ -135,8 +135,8 @@ jsonnet_to_json_test( name = "tla_code_library_test", size = "small", src = "tla_code_library.jsonnet", - tla_code_libraries = { ":code_library_lib": "tla_code" }, golden = "tla_code_library_golden.json", + tla_code_libraries = {":code_library_lib": "tla_code"}, ) jsonnet_to_json_test( @@ -342,19 +342,19 @@ jsonnet_library( srcs = [ "extvar.libsonnet", ], + ext_code_envs = ["MYJSONNET"], + ext_code_libraries = { + ":code_library_lib": "ext_lib", + }, + ext_str_envs = ["MYTEST"], + ext_str_files = { + ":test_str_files": "str_files", + }, ext_strs = { "non_stamp": "non_stamp", "mydefine": "$(mydefine)", "k8s": "{STABLE_K8S_CLUSTER}", }, - ext_str_files = { - ":test_str_files": "str_files" - }, - ext_code_envs = ["MYJSONNET"], - ext_str_envs = ["MYTEST"], - ext_code_libraries = { - ":code_library_lib": "ext_lib" - }, ) jsonnet_to_json_test( @@ -368,14 +368,14 @@ jsonnet_to_json_test( jsonnet_to_json( name = "extvar-lib-json", - outs = ["ext-lib.json"], src = "extvar-library.jsonnet", - deps = [ - ":extvar-lib", - ], + outs = ["ext-lib.json"], + ext_code_envs = ["MYJSONNET"], + ext_str_envs = ["MYTEST"], ext_strs = { "non_stamp": "non_stamp", }, - ext_code_envs = ["MYJSONNET"], - ext_str_envs = ["MYTEST"], + deps = [ + ":extvar-lib", + ], ) diff --git a/jsonnet/jsonnet.bzl b/jsonnet/jsonnet.bzl index 21555b7..f626a70 100644 --- a/jsonnet/jsonnet.bzl +++ b/jsonnet/jsonnet.bzl @@ -92,31 +92,50 @@ def _setup_deps(deps, tla_code_libraries = {}, ext_code_libraries = {}, transiti transitive_extvars = transitive_extvars, ) -def _make_extvar_dict(label, - ext_code, - ext_code_envs, - ext_code_files, - ext_code_libraries, - ext_str_envs, - ext_str_files, - ext_strs, - ): +def _make_extvar_dict( + label, + ext_code, + ext_code_envs, + ext_code_files, + ext_code_libraries, + ext_str_envs, + ext_str_files, + ext_strs): + """Transforms input ext_* attributes and builds a transitive_extvars dict + + Args: + label: Label to track source of ext var for making debugging messages + ext_code: Dict of variable names to code, maps from ctx.attr.ext_code + ext_code_envs: List of variable names that map to environment variables, maps from ctx.attr.ext_code_envs + ext_code_files: Dict of Label or File to variable names, maps from ctx.attr.ext_code_files + ext_code_libraries: Dict of Label to variable names, maps from ctx.attr.ext_code_libraries + ext_str_envs: List of variable names to strings from environment, maps from ctx.attr.ext_str_envs + ext_str_files: Dict of Label or File to variable names, maps from ctx.attr.ext_str_files + ext_strs: Dict of variable names to strings, maps from ctx.attr.ext_strs + + Returns: + Dictionary with keys are variable names, and values a dict containing + type: The kind of extvar it is from and maps to a ctx.attr, e.g. string, code, string_file, etc. + value: The string, code, or File depending on type + sources: List of labels that define the extvar + """ extvars = dict() + label = str(label) for key, code in ext_code.items(): if key in extvars: fail("duplicate extVar '{}': {}".format(key, extvars.keys())) extvars[key] = { - 'value': code, - 'type': 'code', - 'sources': [label], + "value": code, + "type": "code", + "sources": [label], } for key in ext_code_envs: if key in extvars: fail("duplicate extVar '{}': {}".format(key, extvars.keys())) extvars[key] = { - 'value': '', - 'type': 'code_env', - 'sources': [label], + "value": "", + "type": "code_env", + "sources": [label], } for file, key in ext_code_files.items(): if key in extvars: @@ -127,25 +146,25 @@ def _make_extvar_dict(label, files = file[DefaultInfo].files.to_list() extvars[key] = { - 'value': files, - 'type': 'code_file', - 'sources': [label], + "value": files, + "type": "code_file", + "sources": [label], } for file, key in ext_code_libraries.items(): if key in extvars: fail("duplicate extVar '{}': {}".format(key, extvars.keys())) extvars[key] = { - 'value': file[DefaultInfo].files.to_list(), - 'type': 'code_library', - 'sources': [label], + "value": file[DefaultInfo].files.to_list(), + "type": "code_library", + "sources": [label], } for key in ext_str_envs: if key in extvars: fail("duplicate extVar '{}': {}".format(key, extvars.keys())) extvars[key] = { - 'value': '', - 'type': 'string_env', - 'sources': [label], + "value": "", + "type": "string_env", + "sources": [label], } for file, key in ext_str_files.items(): if key in extvars: @@ -156,36 +175,45 @@ def _make_extvar_dict(label, files = file[DefaultInfo].files.to_list() extvars[key] = { - 'value': files, - 'type': 'string_file', - 'sources': [label], + "value": files, + "type": "string_file", + "sources": [label], } for key, val in ext_strs.items(): if key in extvars: fail("duplicate extVar '{}': {}".format(key, extvars.keys())) extvars[key] = { - 'value': val, - 'type': 'string', - 'sources': [label], + "value": val, + "type": "string", + "sources": [label], } return extvars -def _extvar_to_arguments(transitive_extvars, short_path=False): +def _extvar_to_arguments(transitive_extvars, short_path = False): + """Converts an transitive_extvars to command line arguments + + Args: + transitive_extvars: dict of extvar from _make_extvar_dict + short_path: Boolean - if the short_path of files should be used + + Returns: + List of strings of arguments for extvars + """ args = [] for key, val in transitive_extvars.items(): - if val['type'] == 'string': - args.append("--ext-str %s=%s" % (_quote(key), _quote(val['value']))) - elif val['type'] == 'string_env': + if val["type"] == "string": + args.append("--ext-str %s=%s" % (_quote(key), _quote(val["value"]))) + elif val["type"] == "string_env": args.append("--ext-str %s" % _quote(key)) - elif val['type'] == 'string_file': - file = val['value'][0] + elif val["type"] == "string_file": + file = val["value"][0] args.append("--ext-str-file %s=%s" % (_quote(key), _quote(file.short_path if short_path else file.path))) - elif val['type'] == 'code': - args.append("--ext-code %s=%s" % (_quote(key), _quote(val['value']))) - elif val['type'] == 'code_env': + elif val["type"] == "code": + args.append("--ext-code %s=%s" % (_quote(key), _quote(val["value"]))) + elif val["type"] == "code_env": args.append("--ext-code %s" % _quote(key)) - elif val['type'] == 'code_library' or val['type'] == 'code_file': - file = val['value'][0] + elif val["type"] == "code_library" or val["type"] == "code_file": + file = val["value"][0] args.append("--ext-code-file %s=%s" % (_quote(key), _quote(file.short_path if short_path else file.path))) else: fail("The {} key has an unknown extvar type {}: {}".format(key, val["type"], val["sources"])) @@ -194,7 +222,17 @@ def _extvar_to_arguments(transitive_extvars, short_path=False): def _merge_extvars(left, right): """Merges two extvar dicts together - + + In the case of duplicates extvar (keys of the dict): + 1. If type & value match: the inner sources are merged + 2. If type or value mismatches: raises an error + + Args: + left: A dictonary made from _make_ext_dict + right: A dictonary made from _make_ext_dict + + Returns: + A _make_ext_dict compatible dict """ result = dict(left) @@ -202,19 +240,21 @@ def _merge_extvars(left, right): # Check if the variable name has been used already if var in left: left_val = left[var] + # Check if is the same type & value - if left_val['type'] != right_val['type']: + if left_val["type"] != right_val["type"]: # If the types are different fail("extvar {} is defined in multiple places with different types: {}" - .format(var, left_val['sources'] + right_val['sources'])) - elif left_val['value'] != right_val['value']: + .format(var, left_val["sources"] + right_val["sources"])) + elif left_val["value"] != right_val["value"]: fail("extvar {} is defined in multiple places with different values: {}" - .format(var, left_val['sources'] + right_val['sources'])) + .format(var, left_val["sources"] + right_val["sources"])) else: # type & value match! - # Collect the sources to provide better error messages - result[var]['sources'].extend(right_val['sources']) + # Collect the sources to provide better error messages if there ever is a mismatch + result[var]["sources"].extend(right_val["sources"]) else: + # Simple case, right side has a new variable result[var] = right_val return result @@ -234,13 +274,13 @@ def _jsonnet_library_impl(ctx): depinfo = _setup_deps( ctx.attr.deps, - ext_code_libraries=ctx.attr.ext_code_libraries, - transitive_extvars=transitive_extvars + ext_code_libraries = ctx.attr.ext_code_libraries, + transitive_extvars = transitive_extvars, ) sources = depset( ctx.files.srcs + ctx.files.ext_code_files + ctx.files.ext_str_files, - transitive = [ depinfo.transitive_sources ] + transitive = [depinfo.transitive_sources], ) imports = depset( _get_import_paths(ctx.label, ctx.files.srcs, ctx.attr.imports, False), @@ -704,20 +744,30 @@ _jsonnet_library_attrs = { doc = "List of `.jsonnet` files that comprises this Jsonnet library", allow_files = _JSONNET_FILETYPE, ), - "ext_code": attr.string_dict(), - "ext_code_envs": attr.string_list(), + "ext_code": attr.string_dict( + doc = "Include code from the dict value via extvar. Variable name matches the key" + ), + "ext_code_envs": attr.string_list( + doc = "Include code from an environment variable via extvar. Variable name matches the environment variable name" + ), "ext_code_files": attr.label_keyed_string_dict( + doc = "Include code from a file from dict key via extvar. Variable name matches the value", allow_files = True, ), "ext_code_libraries": attr.label_keyed_string_dict( doc = "Include jsonnet_library as an extvar with the key value", providers = [JsonnetLibraryInfo], ), - "ext_str_envs": attr.string_list(), + "ext_str_envs": attr.string_list( + doc = "Include string from an environment variable via extvar. Variable name matches the environment variable name" + ), "ext_str_files": attr.label_keyed_string_dict( + doc = "Include string from a file from dict key via extvar. Variable name matches the value", allow_files = True, ), - "ext_strs": attr.string_dict(), + "ext_strs": attr.string_dict( + doc = "Include string from the dict value via extvar. Variable name matches the key" + ), } jsonnet_library = rule( From 90030f6b97b2750042881fa78df40c8ccc0c2c90 Mon Sep 17 00:00:00 2001 From: David Barnett Date: Mon, 15 Sep 2025 07:48:44 +1000 Subject: [PATCH 3/5] ai feedback --- jsonnet/jsonnet.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jsonnet/jsonnet.bzl b/jsonnet/jsonnet.bzl index f626a70..88833e4 100644 --- a/jsonnet/jsonnet.bzl +++ b/jsonnet/jsonnet.bzl @@ -306,7 +306,7 @@ def _jsonnet_library_impl(ctx): imports = imports, short_imports = short_imports, transitive_jsonnet_files = sources, - transitive_extvars = transitive_extvars, + transitive_extvars = depinfo.transitive_extvars, ), ] From 9271e9a28deb9ebcf290243304854d4902850e3c Mon Sep 17 00:00:00 2001 From: David Barnett Date: Tue, 16 Sep 2025 07:45:52 +1000 Subject: [PATCH 4/5] ai feedback and stamping fix --- jsonnet/jsonnet.bzl | 110 +++++++++++++++----------------------------- 1 file changed, 38 insertions(+), 72 deletions(-) diff --git a/jsonnet/jsonnet.bzl b/jsonnet/jsonnet.bzl index 88833e4..c9c8c23 100644 --- a/jsonnet/jsonnet.bzl +++ b/jsonnet/jsonnet.bzl @@ -122,73 +122,39 @@ def _make_extvar_dict( extvars = dict() label = str(label) for key, code in ext_code.items(): - if key in extvars: - fail("duplicate extVar '{}': {}".format(key, extvars.keys())) - extvars[key] = { - "value": code, - "type": "code", - "sources": [label], - } + _make_extvar_dict_update(extvars, "code", key, code, label) for key in ext_code_envs: - if key in extvars: - fail("duplicate extVar '{}': {}".format(key, extvars.keys())) - extvars[key] = { - "value": "", - "type": "code_env", - "sources": [label], - } + _make_extvar_dict_update(extvars, "code_env", key, None, label) for file, key in ext_code_files.items(): - if key in extvars: - fail("duplicate extVar '{}': {}".format(key, extvars.keys())) - - files = [file] - if type(file) != "File": - files = file[DefaultInfo].files.to_list() - - extvars[key] = { - "value": files, - "type": "code_file", - "sources": [label], - } + _make_extvar_dict_update(extvars, "code_file", key, file, label) for file, key in ext_code_libraries.items(): - if key in extvars: - fail("duplicate extVar '{}': {}".format(key, extvars.keys())) - extvars[key] = { - "value": file[DefaultInfo].files.to_list(), - "type": "code_library", - "sources": [label], - } + _make_extvar_dict_update(extvars, "code_library", key, file, label) for key in ext_str_envs: - if key in extvars: - fail("duplicate extVar '{}': {}".format(key, extvars.keys())) - extvars[key] = { - "value": "", - "type": "string_env", - "sources": [label], - } - for file, key in ext_str_files.items(): - if key in extvars: - fail("duplicate extVar '{}': {}".format(key, extvars.keys())) - - files = [file] - if type(file) != "File": - files = file[DefaultInfo].files.to_list() - - extvars[key] = { - "value": files, - "type": "string_file", - "sources": [label], - } + _make_extvar_dict_update(extvars, "string_env", key, None, label) + for val, key in ext_str_files.items(): + _make_extvar_dict_update(extvars, "string_file", key, val, label) for key, val in ext_strs.items(): - if key in extvars: - fail("duplicate extVar '{}': {}".format(key, extvars.keys())) - extvars[key] = { - "value": val, - "type": "string", - "sources": [label], - } + _make_extvar_dict_update(extvars, "string", key, val, label) return extvars +def _make_extvar_dict_update(extvars, extvar_type, key, val, label): + if key in extvars: + fail("duplicate extvar '{}' of type {} and {}" + .format(key, extvar_type, extvars[key]["type"])) + + if type(val) == "string" or type(val) == "File" or val == None: + pass + elif type(val) == "Target": + val = val[DefaultInfo].files.to_list()[0] + else: + fail("unknown type of value {} for {} in {}".format(type(val), key, label)) + + extvars.update([[key, { + "value": val, + "type": extvar_type, + "sources": [label], + }]]) + def _extvar_to_arguments(transitive_extvars, short_path = False): """Converts an transitive_extvars to command line arguments @@ -206,14 +172,14 @@ def _extvar_to_arguments(transitive_extvars, short_path = False): elif val["type"] == "string_env": args.append("--ext-str %s" % _quote(key)) elif val["type"] == "string_file": - file = val["value"][0] + file = val["value"] args.append("--ext-str-file %s=%s" % (_quote(key), _quote(file.short_path if short_path else file.path))) elif val["type"] == "code": args.append("--ext-code %s=%s" % (_quote(key), _quote(val["value"]))) elif val["type"] == "code_env": args.append("--ext-code %s" % _quote(key)) elif val["type"] == "code_library" or val["type"] == "code_file": - file = val["value"][0] + file = val["value"] args.append("--ext-code-file %s=%s" % (_quote(key), _quote(file.short_path if short_path else file.path))) else: fail("The {} key has an unknown extvar type {}: {}".format(key, val["type"], val["sources"])) @@ -384,6 +350,12 @@ def _jsonnet_to_json_impl(ctx): jsonnet_tla_code_files = ctx.attr.tla_code_files jsonnet_tla_code_libraries = ctx.attr.tla_code_libraries + jsonnet_ext_strs, strs_stamp_inputs = _make_stamp_resolve(ctx.attr.ext_strs, ctx, False) + jsonnet_ext_code, code_stamp_inputs = _make_stamp_resolve(ctx.attr.ext_code, ctx, False) + jsonnet_tla_strs, tla_strs_stamp_inputs = _make_stamp_resolve(ctx.attr.tla_strs, ctx, False) + jsonnet_tla_code, tla_code_stamp_inputs = _make_stamp_resolve(ctx.attr.tla_code, ctx, False) + stamp_inputs = strs_stamp_inputs + code_stamp_inputs + tla_strs_stamp_inputs + tla_code_stamp_inputs + transitive_extvars = _make_extvar_dict( ctx.label, jsonnet_ext_code, @@ -402,12 +374,6 @@ def _jsonnet_to_json_impl(ctx): transitive_extvars, ) - jsonnet_ext_strs, strs_stamp_inputs = _make_stamp_resolve(ctx.attr.ext_strs, ctx, False) - jsonnet_ext_code, code_stamp_inputs = _make_stamp_resolve(ctx.attr.ext_code, ctx, False) - jsonnet_tla_strs, tla_strs_stamp_inputs = _make_stamp_resolve(ctx.attr.tla_strs, ctx, False) - jsonnet_tla_code, tla_code_stamp_inputs = _make_stamp_resolve(ctx.attr.tla_code, ctx, False) - stamp_inputs = strs_stamp_inputs + code_stamp_inputs + tla_strs_stamp_inputs + tla_code_stamp_inputs - if len(jsonnet_ext_str_file_vars) != len(jsonnet_ext_str_files): fail("Mismatch of ext_str_file_vars ({}) to ext_str_files ({})".format(jsonnet_ext_str_file_vars, jsonnet_ext_str_files)) @@ -745,10 +711,10 @@ _jsonnet_library_attrs = { allow_files = _JSONNET_FILETYPE, ), "ext_code": attr.string_dict( - doc = "Include code from the dict value via extvar. Variable name matches the key" + doc = "Include code from the dict value via extvar. Variable name matches the key", ), "ext_code_envs": attr.string_list( - doc = "Include code from an environment variable via extvar. Variable name matches the environment variable name" + doc = "Include code from an environment variable via extvar. Variable name matches the environment variable name", ), "ext_code_files": attr.label_keyed_string_dict( doc = "Include code from a file from dict key via extvar. Variable name matches the value", @@ -759,14 +725,14 @@ _jsonnet_library_attrs = { providers = [JsonnetLibraryInfo], ), "ext_str_envs": attr.string_list( - doc = "Include string from an environment variable via extvar. Variable name matches the environment variable name" + doc = "Include string from an environment variable via extvar. Variable name matches the environment variable name", ), "ext_str_files": attr.label_keyed_string_dict( doc = "Include string from a file from dict key via extvar. Variable name matches the value", allow_files = True, ), "ext_strs": attr.string_dict( - doc = "Include string from the dict value via extvar. Variable name matches the key" + doc = "Include string from the dict value via extvar. Variable name matches the key", ), } From 339b171ba94df810d031a158b405a4187ac09d2e Mon Sep 17 00:00:00 2001 From: David Barnett Date: Mon, 22 Sep 2025 07:11:30 +1000 Subject: [PATCH 5/5] feedback, remove stringly types --- jsonnet/jsonnet.bzl | 110 +++++++++++++++++++++++++++++--------------- 1 file changed, 73 insertions(+), 37 deletions(-) diff --git a/jsonnet/jsonnet.bzl b/jsonnet/jsonnet.bzl index c9c8c23..09ee8e0 100644 --- a/jsonnet/jsonnet.bzl +++ b/jsonnet/jsonnet.bzl @@ -59,6 +59,7 @@ def _setup_deps(deps, tla_code_libraries = {}, ext_code_libraries = {}, transiti deps: List of deps labels from ctx.attr.deps. tla_code_libraries: Dict of labels to names from ctx.attr.tla_code_files. ext_code_libraries: List of deps labels from ctx.attr.ext_code_files. + transitive_extvars: Dict of extvar to values build from _make_extvar_dict Returns: Returns a struct containing the following fields: @@ -69,6 +70,9 @@ def _setup_deps(deps, tla_code_libraries = {}, ext_code_libraries = {}, transiti short_imports: Depset of Strings containing import flags set by transitive dependency targets, when invoking Jsonnet as part of a test where dependencies are stored in runfiles. + transitive_extvars: Dict of extvar to values that has merged the + input value with all extvars of its depdencies. + """ transitive_sources = [] imports = [] @@ -115,42 +119,72 @@ def _make_extvar_dict( Returns: Dictionary with keys are variable names, and values a dict containing - type: The kind of extvar it is from and maps to a ctx.attr, e.g. string, code, string_file, etc. + type: The type of extvar it will be in jsonnet: string or code value: The string, code, or File depending on type sources: List of labels that define the extvar """ extvars = dict() label = str(label) - for key, code in ext_code.items(): - _make_extvar_dict_update(extvars, "code", key, code, label) - for key in ext_code_envs: - _make_extvar_dict_update(extvars, "code_env", key, None, label) - for file, key in ext_code_files.items(): - _make_extvar_dict_update(extvars, "code_file", key, file, label) - for file, key in ext_code_libraries.items(): - _make_extvar_dict_update(extvars, "code_library", key, file, label) - for key in ext_str_envs: - _make_extvar_dict_update(extvars, "string_env", key, None, label) - for val, key in ext_str_files.items(): - _make_extvar_dict_update(extvars, "string_file", key, val, label) - for key, val in ext_strs.items(): + + # extvar_lists is a list of tuple (extvar: str, value: None | str | File | JsonnetInfo, extvar_type: str) + # The `None` value are used by environment + # Collect all the Code extvars + # ext_code, dict[extvar, str_value] + extvar_code_lists = zip(ext_code.keys(), ext_code.values()) + + # ext_code_envs, list[extvar] + extvar_code_lists.extend(zip(ext_code_envs, [None] * len(ext_code_envs))) + + # ext_code_files, dict[label, extvar] + extvar_code_lists.extend(zip(ext_code_files.values(), ext_code_files.keys())) + + # ext_code_libraries, dict[label, extvar] + extvar_code_lists.extend(zip(ext_code_libraries.values(), ext_code_libraries.keys())) + + for key, val in extvar_code_lists: + _make_extvar_dict_update(extvars, "code", key, val, label) + + # Collect all of the String extvars + # ext_str_envs, list[extvar] + extvar_str_lists = zip(ext_str_envs, [None] * len(ext_str_envs)) + + # ext_str_files, dict[label, extvar] + extvar_str_lists.extend(zip(ext_str_files.values(), ext_str_files.keys())) + + # ext_strs, dict[extvar, str] + extvar_str_lists.extend(zip(ext_strs.keys(), ext_strs.values())) + + for key, val in extvar_str_lists: _make_extvar_dict_update(extvars, "string", key, val, label) + return extvars -def _make_extvar_dict_update(extvars, extvar_type, key, val, label): - if key in extvars: +def _make_extvar_dict_update(extvars, extvar_type, extvar_name, value, label): + """Adds an entry to a given extrvars dict and validates its uniqueness + + Args: + extvars: Dict of extvars to be added to + extvar_type: String of either "string" or "code" + extvar_name: String of the extvar variable name + value: Either a None, string, File or Target + label: String of the package this extvar is defined in + + Returns: + None, modifies the given extvars input in-place + """ + if extvar_name in extvars: fail("duplicate extvar '{}' of type {} and {}" - .format(key, extvar_type, extvars[key]["type"])) + .format(extvar_name, extvar_type, extvars[extvar_name]["type"])) - if type(val) == "string" or type(val) == "File" or val == None: + if type(value) == "string" or type(value) == "File" or value == None: pass - elif type(val) == "Target": - val = val[DefaultInfo].files.to_list()[0] + elif type(value) == "Target": + value = value[DefaultInfo].files.to_list()[0] else: - fail("unknown type of value {} for {} in {}".format(type(val), key, label)) + fail("unknown type of value {} for {} in {}".format(type(value), extvar_name, label)) - extvars.update([[key, { - "value": val, + extvars.update([[extvar_name, { + "value": value, "type": extvar_type, "sources": [label], }]]) @@ -167,22 +201,24 @@ def _extvar_to_arguments(transitive_extvars, short_path = False): """ args = [] for key, val in transitive_extvars.items(): - if val["type"] == "string": - args.append("--ext-str %s=%s" % (_quote(key), _quote(val["value"]))) - elif val["type"] == "string_env": - args.append("--ext-str %s" % _quote(key)) - elif val["type"] == "string_file": - file = val["value"] - args.append("--ext-str-file %s=%s" % (_quote(key), _quote(file.short_path if short_path else file.path))) - elif val["type"] == "code": - args.append("--ext-code %s=%s" % (_quote(key), _quote(val["value"]))) - elif val["type"] == "code_env": - args.append("--ext-code %s" % _quote(key)) - elif val["type"] == "code_library" or val["type"] == "code_file": + # The --ext-str-* and --ext-code-* flag families are interchangable, + # so the `type` is used to determine which to use. + flag_type = "str" if val["type"] == "string" else val["type"] + + # Each different type of value is formatted in the flags differently + if val["value"] == None: + # Environment flags + args.append("--ext-%s %s" % (flag_type, _quote(key))) + elif type(val["value"]) == "string": + # String flags + args.append("--ext-%s %s=%s" % (flag_type, _quote(key), _quote(val["value"]))) + elif type(val["value"]) == "File": + # Files and library flags file = val["value"] - args.append("--ext-code-file %s=%s" % (_quote(key), _quote(file.short_path if short_path else file.path))) + file_path = file.short_path if short_path else file.path + args.append("--ext-%s-file %s=%s" % (flag_type, _quote(key), _quote(file_path))) else: - fail("The {} key has an unknown extvar type {}: {}".format(key, val["type"], val["sources"])) + fail("The {} key has an unknown extvar type {}: {}".format(key, type(val["value"]), val["sources"])) return args