From 81b437e8c2d3b51687d9fa20b5025124a4166ca3 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Thu, 6 Mar 2025 09:57:41 +0100 Subject: [PATCH 01/52] update definitions --- src/pynxtools/nexus-version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/nexus-version.txt b/src/pynxtools/nexus-version.txt index 5bef17fbf..2436a229a 100644 --- a/src/pynxtools/nexus-version.txt +++ b/src/pynxtools/nexus-version.txt @@ -1 +1 @@ -v2022.07-1521-gf7ba53f4 \ No newline at end of file +v2024.02-1728-g3ffaf5d2 \ No newline at end of file From 5b7a4f5cc8bf814b3c42dba2bffdccc08c4336e0 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Thu, 6 Mar 2025 17:12:08 +0100 Subject: [PATCH 02/52] change plugin branches for testing --- .github/workflows/plugin_test.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/plugin_test.yaml b/.github/workflows/plugin_test.yaml index 32fd9bbfd..eeceaf6b6 100644 --- a/.github/workflows/plugin_test.yaml +++ b/.github/workflows/plugin_test.yaml @@ -24,25 +24,25 @@ jobs: branch: main tests_to_run: tests/. - plugin: pynxtools-ellips - branch: main + branch: update tests_to_run: tests/. - plugin: pynxtools-em branch: main tests_to_run: tests/. - plugin: pynxtools-igor branch: main - tests_to_run: tests/. + tests_to_run: tests/. - plugin: pynxtools-mpes branch: main tests_to_run: tests/. - plugin: pynxtools-raman - branch: main + branch: update tests_to_run: tests/. - plugin: pynxtools-spm - branch: main + branch: FixReaderAlngAppDef tests_to_run: tests/. - plugin: pynxtools-xps - branch: main + branch: update-to-new-nxmpes tests_to_run: tests/. - plugin: pynxtools-xrd branch: main From e19509ebb9c5a2e3a23779c83707c389b9ac2a96 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Fri, 7 Mar 2025 11:40:41 +0100 Subject: [PATCH 03/52] update definitions once more --- src/pynxtools/nexus-version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/nexus-version.txt b/src/pynxtools/nexus-version.txt index 2436a229a..aea1d0c9b 100644 --- a/src/pynxtools/nexus-version.txt +++ b/src/pynxtools/nexus-version.txt @@ -1 +1 @@ -v2024.02-1728-g3ffaf5d2 \ No newline at end of file +v2024.02-1741-g9af50e9f \ No newline at end of file From 912e60c3ef9ad9de6d9c1c2fe5a29fafffad8d53 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Fri, 7 Mar 2025 11:44:43 +0100 Subject: [PATCH 04/52] initial attempt at nameType in validation --- src/pynxtools/dataconverter/nexus_tree.py | 35 +++++++++++++++++++---- src/pynxtools/dataconverter/validation.py | 24 +++++++++++++--- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/pynxtools/dataconverter/nexus_tree.py b/src/pynxtools/dataconverter/nexus_tree.py index bbba22c09..d9ff05d97 100644 --- a/src/pynxtools/dataconverter/nexus_tree.py +++ b/src/pynxtools/dataconverter/nexus_tree.py @@ -111,6 +111,9 @@ class NexusNode(NodeMixin): The name of the node. type (Literal["group", "field", "attribute", "choice"]): The type of the node, e.g., xml tag in the nxdl file. + name_type (Optional["specified", "any", "partial"]): + The nameType of the node. + Defaults to "specified". optionality (Literal["required", "recommended", "optional"], optional): The optionality of the node. This is automatically set on init (in the respective subclasses) @@ -118,8 +121,8 @@ class NexusNode(NodeMixin): Defaults to "required". variadic (bool): True if the node name is variadic and can be matched against multiple names. - This is set automatically on init and will be True if the name contains - any uppercase characets and False otherwise. + This is set automatically on init and will be True if the `nameTYPE` is "any" + or "partial" and False otherwise. Defaults to False. inheritance (List[InstanceOf[ET._Element]]): The inheritance chain of the node. @@ -145,6 +148,7 @@ class NexusNode(NodeMixin): name: str type: Literal["group", "field", "attribute", "choice"] + name_type: Optional[Literal["specified", "any", "partial"]] = "specified" optionality: Literal["required", "recommended", "optional"] = "required" variadic: bool = False inheritance: List[ET._Element] @@ -177,6 +181,7 @@ def __init__( self, name: str, type: Literal["group", "field", "attribute", "choice"], + name_type: Optional[Literal["specified", "any", "partial"]] = "specified", optionality: Literal["required", "recommended", "optional"] = "required", variadic: Optional[bool] = None, parent: Optional["NexusNode"] = None, @@ -185,8 +190,9 @@ def __init__( super().__init__() self.name = name self.type = type + self.name_type = name_type self.optionality = optionality - self.variadic = contains_uppercase(self.name) + self.variadic = self._is_variadic(self.name, self.name_type) if variadic is not None: self.variadic = variadic if inheritance is not None: @@ -197,6 +203,14 @@ def __init__( self.is_a = [] self.parent_of = [] + def _is_variadic(self, name: str, name_type: str) -> bool: + """ + Determine if a name is variadic based on its nameType. + """ + if name: + return True if name_type in ("any", "partial") else False + return True + def _construct_inheritance_chain_from_parent(self): """ Builds the inheritance chain of the current node based on the parent node. @@ -525,21 +539,31 @@ def add_node_from(self, xml_elem: ET._Element) -> Optional["NexusNode"]: """ default_optionality = "required" if is_appdef(xml_elem) else "optional" tag = remove_namespace_from_tag(xml_elem.tag) + + name_type = xml_elem.attrib.get("nameType", "specified") + if tag in ("field", "attribute"): name = xml_elem.attrib.get("name") + current_elem = NexusEntity( parent=self, name=name, + name_type=name_type, type=tag, optionality=default_optionality, ) elif tag == "group": - name = xml_elem.attrib.get("name", xml_elem.attrib["type"][2:].upper()) + name = xml_elem.attrib.get("name") + if not name: + name = xml_elem.attrib["type"][2:].upper() + name_type = None + inheritance_chain = self._build_inheritance_chain(xml_elem) current_elem = NexusGroup( parent=self, type=tag, name=name, + name_type=name_type, nx_class=xml_elem.attrib["type"], inheritance=inheritance_chain, optionality=default_optionality, @@ -548,7 +572,7 @@ def add_node_from(self, xml_elem: ET._Element) -> Optional["NexusNode"]: current_elem = NexusChoice( parent=self, name=xml_elem.attrib["name"], - variadic=contains_uppercase(xml_elem.attrib["name"]), + name_type=name_type, optionality=default_optionality, ) else: @@ -910,6 +934,7 @@ def add_children_to(parent: NexusNode, xml_elem: ET._Element) -> None: name=appdef_xml_root.attrib["name"], nx_class="NXroot", type="group", + name_type="specified", optionality="required", variadic=False, parent=None, diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index 4b599b43a..050a0f42f 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -134,13 +134,14 @@ def split_class_and_name_of(name: str) -> Tuple[Optional[str], str]: ), f"{name_match.group(2)}{'' if prefix is None else prefix}" -def best_namefit_of(name: str, keys: Iterable[str]) -> Optional[str]: +def best_namefit_of(name: str, keys: Iterable[str], name_type: str) -> Optional[str]: """ Get the best namefit of `name` in `keys`. Args: name (str): The name to fit against the keys. keys (Iterable[str]): The keys to fit `name` against. + name_type (str): nameType of the concept being fitted Returns: Optional[str]: The best fitting key. None if no fit was found. @@ -155,9 +156,14 @@ def best_namefit_of(name: str, keys: Iterable[str]) -> Optional[str]: if nx_name is not None and nx_name in keys: return nx_name + name_any = True if name_type == "any" else False + name_partial = True if name_type == "partial" else False + best_match, score = max( - map(lambda x: (x, get_nx_namefit(name2fit, x)), keys), key=lambda x: x[1] + map(lambda x: (x, get_nx_namefit(name2fit, x, name_any, name_partial)), keys), + key=lambda x: x[1], ) + print(nx_name, name2fit, best_match, score) if score < 0: return None @@ -188,6 +194,9 @@ def validate_dict_against( def get_variations_of(node: NexusNode, keys: Mapping[str, Any]) -> List[str]: if not node.variadic: + print(node.name, node.name_type, node.variadic) + if hasattr(node, "nx_class"): + print(f"{convert_nexus_to_caps(node.nx_class)}[{node.name}]") # in keys if node.name in keys: return [node.name] elif ( @@ -198,14 +207,21 @@ def get_variations_of(node: NexusNode, keys: Mapping[str, Any]) -> List[str]: variations = [] for key in keys: + # print(key) nx_name, name2fit = split_class_and_name_of(key) if node.type == "attribute": # Remove the starting @ from attributes name2fit = name2fit[1:] if name2fit.startswith("@") else name2fit if nx_name is not None and nx_name != node.name: continue + name_any = True if node.name_type == "any" else False + name_partial = True if node.name_type == "partial" else False + + # if nx_name and "ENTRY" in nx_name: + # print(nx_name, name2fit, key) + if ( - get_nx_namefit(name2fit, node.name) >= 0 + get_nx_namefit(name2fit, node.name, name_any, name_partial) >= 0 and key not in node.parent.get_all_direct_children_names() ): variations.append(key) @@ -511,7 +527,7 @@ def is_documented(key: str, node: NexusNode) -> bool: for name in key[1:].replace("@", "").split("/"): children = node.get_all_direct_children_names() - best_name = best_namefit_of(name, children) + best_name = best_namefit_of(name, children, node.name_type) if best_name is None: return False From a64af5e6fc8c13495f361542513568b34642c1cf Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Fri, 7 Mar 2025 16:09:40 +0100 Subject: [PATCH 05/52] update defs again --- src/pynxtools/nexus-version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/nexus-version.txt b/src/pynxtools/nexus-version.txt index aea1d0c9b..2e5e8912a 100644 --- a/src/pynxtools/nexus-version.txt +++ b/src/pynxtools/nexus-version.txt @@ -1 +1 @@ -v2024.02-1741-g9af50e9f \ No newline at end of file +v2024.02-1746-gc8c2aea8 \ No newline at end of file From 882ff00b8d0c158c378a3153be60e1c6c96ae0d3 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Mon, 10 Mar 2025 15:50:23 +0100 Subject: [PATCH 06/52] update defs --- src/pynxtools/nexus-version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/nexus-version.txt b/src/pynxtools/nexus-version.txt index 2e5e8912a..3f2a82f53 100644 --- a/src/pynxtools/nexus-version.txt +++ b/src/pynxtools/nexus-version.txt @@ -1 +1 @@ -v2024.02-1746-gc8c2aea8 \ No newline at end of file +v2024.02-1753-g3f66054d \ No newline at end of file From 971db9af2dd72b2f730c3d8f207bada29745a2d8 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 11 Mar 2025 09:36:00 +0100 Subject: [PATCH 07/52] nameType fixes, remove prints --- src/pynxtools/dataconverter/nexus_tree.py | 2 +- src/pynxtools/dataconverter/validation.py | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/pynxtools/dataconverter/nexus_tree.py b/src/pynxtools/dataconverter/nexus_tree.py index d9ff05d97..ae597d361 100644 --- a/src/pynxtools/dataconverter/nexus_tree.py +++ b/src/pynxtools/dataconverter/nexus_tree.py @@ -208,7 +208,7 @@ def _is_variadic(self, name: str, name_type: str) -> bool: Determine if a name is variadic based on its nameType. """ if name: - return True if name_type in ("any", "partial") else False + return False if name_type == "specified" else True return True def _construct_inheritance_chain_from_parent(self): diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index 050a0f42f..7bb56bbd7 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -163,7 +163,6 @@ def best_namefit_of(name: str, keys: Iterable[str], name_type: str) -> Optional[ map(lambda x: (x, get_nx_namefit(name2fit, x, name_any, name_partial)), keys), key=lambda x: x[1], ) - print(nx_name, name2fit, best_match, score) if score < 0: return None @@ -194,9 +193,6 @@ def validate_dict_against( def get_variations_of(node: NexusNode, keys: Mapping[str, Any]) -> List[str]: if not node.variadic: - print(node.name, node.name_type, node.variadic) - if hasattr(node, "nx_class"): - print(f"{convert_nexus_to_caps(node.nx_class)}[{node.name}]") # in keys if node.name in keys: return [node.name] elif ( @@ -207,19 +203,15 @@ def get_variations_of(node: NexusNode, keys: Mapping[str, Any]) -> List[str]: variations = [] for key in keys: - # print(key) nx_name, name2fit = split_class_and_name_of(key) if node.type == "attribute": # Remove the starting @ from attributes name2fit = name2fit[1:] if name2fit.startswith("@") else name2fit if nx_name is not None and nx_name != node.name: continue - name_any = True if node.name_type == "any" else False + name_any = True if node.name_type in ("any", None) else False name_partial = True if node.name_type == "partial" else False - # if nx_name and "ENTRY" in nx_name: - # print(nx_name, name2fit, key) - if ( get_nx_namefit(name2fit, node.name, name_any, name_partial) >= 0 and key not in node.parent.get_all_direct_children_names() From d9b7921b0ec39cbaf5325687f0720eb513b75ff8 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 11 Mar 2025 11:33:10 +0100 Subject: [PATCH 08/52] implement test cases for partial nameType and open enums --- src/pynxtools/data/NXtest.nxdl.xml | 18 +++++++++----- tests/dataconverter/test_validation.py | 34 ++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/pynxtools/data/NXtest.nxdl.xml b/src/pynxtools/data/NXtest.nxdl.xml index 8695a20c9..8bf82113c 100644 --- a/src/pynxtools/data/NXtest.nxdl.xml +++ b/src/pynxtools/data/NXtest.nxdl.xml @@ -19,7 +19,7 @@ - + A dummy entry to test optional parent check for a required child. @@ -27,7 +27,7 @@ A dummy entry to test optional parent check for an optional child. - + A dummy entry for a float value. @@ -48,10 +48,16 @@ - - - - + + + + + + + + + + diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index 2c946a3a1..e213857f4 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -42,6 +42,7 @@ def get_data_dict(): "/ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value": "just chars", "/ENTRY[my_entry]/NXODD_name[nxodd_name]/char_value/@units": "", "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type": "2nd type", + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2": "2nd type open", "/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value": "2022-01-22T12:14:12.05018+00:00", "/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value/@units": "", "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/bool_value": True, @@ -143,3 +144,36 @@ def test_validation_shows_warning(caplog, data_dict, error_message): assert not validate_dict_against("NXtest", data_dict)[0] assert error_message in caplog.text + + +@pytest.mark.parametrize( + "data_dict, message, expected_fail", + [ + pytest.param( + alter_dict( + { + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type": "a very different type" + }, + get_data_dict(), + ), + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type should be on of the following strings: ['1st type', '2nd type', '3rd type', '4th type']", + True, + id="closed-enum-with-new-item", + ), + pytest.param( + alter_dict( + { + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2": "a very different type" + }, + get_data_dict(), + ), + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type2 does not match with the enumerated items from the open enumeration: ['1st type open', '2nd type open'].", + False, + id="open-enum-with-new-item", + ), + ], +) +def test_validation_enumeration(caplog, data_dict, message, expected_fail): + with caplog.at_level(logging.WARNING): + assert validate_dict_against("NXtest", data_dict)[0] == (not expected_fail) + assert message in caplog.text From f632f48fdc02c8d2a872cf0c297bd80eb8528076 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 11 Mar 2025 11:36:38 +0100 Subject: [PATCH 09/52] implement open enums in the validation --- src/pynxtools/dataconverter/helpers.py | 47 +++++++++++++---------- src/pynxtools/dataconverter/nexus_tree.py | 8 +++- src/pynxtools/dataconverter/validation.py | 18 ++++++--- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/src/pynxtools/dataconverter/helpers.py b/src/pynxtools/dataconverter/helpers.py index 71d4a4b9f..4a1dfbc6c 100644 --- a/src/pynxtools/dataconverter/helpers.py +++ b/src/pynxtools/dataconverter/helpers.py @@ -48,24 +48,25 @@ class ValidationProblem(Enum): UnitWithoutDocumentation = 1 InvalidEnum = 2 - MissingRequiredGroup = 3 - MissingRequiredField = 4 - MissingRequiredAttribute = 5 - InvalidType = 6 - InvalidDatetime = 7 - IsNotPosInt = 8 - ExpectedGroup = 9 - MissingDocumentation = 10 - MissingUnit = 11 - ChoiceValidationError = 12 - UnitWithoutField = 13 - AttributeForNonExistingField = 14 - BrokenLink = 15 - FailedNamefitting = 16 - NXdataMissingSignalData = 17 - NXdataMissingAxisData = 18 - NXdataAxisMismatch = 19 - KeyToBeRemoved = 20 + OpenEnumWithNewItem = 3 + MissingRequiredGroup = 4 + MissingRequiredField = 5 + MissingRequiredAttribute = 6 + InvalidType = 7 + InvalidDatetime = 8 + IsNotPosInt = 9 + ExpectedGroup = 10 + MissingDocumentation = 11 + MissingUnit = 12 + ChoiceValidationError = 13 + UnitWithoutField = 14 + AttributeForNonExistingField = 15 + BrokenLink = 16 + FailedNamefitting = 17 + NXdataMissingSignalData = 18 + NXdataMissingAxisData = 19 + NXdataAxisMismatch = 20 + KeyToBeRemoved = 21 class Collector: @@ -81,11 +82,15 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar if log_type == ValidationProblem.UnitWithoutDocumentation: logger.warning( - f"The unit, {path} = {value}, is being written but has no documentation" + f"The unit, {path} = {value}, is being written but has no documentation." ) elif log_type == ValidationProblem.InvalidEnum: logger.warning( - f"The value at {path} should be on of the following strings: {value}" + f"The value at {path} should be on of the following strings: {value}." + ) + elif log_type == ValidationProblem.OpenEnumWithNewItem: + logger.info( + f"The value at {path} does not match with the enumerated items from the open enumeration: {value}." ) elif log_type == ValidationProblem.MissingRequiredGroup: logger.warning(f"The required group, {path}, hasn't been supplied.") @@ -122,7 +127,7 @@ def _log(self, path: str, log_type: ValidationProblem, value: Optional[Any], *ar elif log_type == ValidationProblem.MissingRequiredAttribute: logger.warning(f'Missing attribute: "{path}"') elif log_type == ValidationProblem.UnitWithoutField: - logger.warning(f"Unit {path} in dataset without its field {value}") + logger.warning(f"Unit {path} in dataset without its field {value}.") elif log_type == ValidationProblem.AttributeForNonExistingField: logger.warning( f"There were attributes set for the field {path}, " diff --git a/src/pynxtools/dataconverter/nexus_tree.py b/src/pynxtools/dataconverter/nexus_tree.py index ae597d361..88890f978 100644 --- a/src/pynxtools/dataconverter/nexus_tree.py +++ b/src/pynxtools/dataconverter/nexus_tree.py @@ -786,6 +786,7 @@ class NexusEntity(NexusNode): unit: Optional[NexusUnitCategory] = None dtype: NexusType = "NX_CHAR" items: Optional[List[str]] = None + open_enum: bool = False shape: Optional[Tuple[Optional[int], ...]] = None def _set_type(self): @@ -808,7 +809,7 @@ def _set_unit(self): self.unit = elem.attrib["units"] return - def _set_items(self): + def _set_items_and_enum_type(self): """ Sets the enumeration items of the current entity based on the values in the inheritance chain. @@ -818,7 +819,10 @@ def _set_items(self): return for elem in self.inheritance: enum = elem.find(f"nx:enumeration", namespaces=namespaces) + if enum is not None: + if enum.attrib.get("open") == "true": + self.open_enum = True self.items = [] for items in enum.findall(f"nx:item", namespaces=namespaces): self.items.append(items.attrib["value"]) @@ -865,7 +869,7 @@ def __init__(self, **data) -> None: self._construct_inheritance_chain_from_parent() self._set_unit() self._set_type() - self._set_items() + self._set_items_and_enum_type() self._set_optionality() self._set_shape() diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index 7bb56bbd7..f255c7e2a 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -439,11 +439,19 @@ def handle_field(node: NexusNode, keys: Mapping[str, Any], prev_path: str): node.items is not None and mapping[f"{prev_path}/{variant}"] not in node.items ): - collector.collect_and_log( - f"{prev_path}/{variant}", - ValidationProblem.InvalidEnum, - node.items, - ) + if node.open_enum: + collector.collect_and_log( + f"{prev_path}/{variant}", + ValidationProblem.OpenEnumWithNewItem, + node.items, + ) + + else: + collector.collect_and_log( + f"{prev_path}/{variant}", + ValidationProblem.InvalidEnum, + node.items, + ) # Check unit category if node.unit is not None: From 14da1bf672b8fcef1a1188259f479d240e13438c Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 11 Mar 2025 11:37:03 +0100 Subject: [PATCH 10/52] add NX_CHAR_OR_NUMBER to NeXusTree --- src/pynxtools/dataconverter/nexus_tree.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pynxtools/dataconverter/nexus_tree.py b/src/pynxtools/dataconverter/nexus_tree.py index 88890f978..ddf343b21 100644 --- a/src/pynxtools/dataconverter/nexus_tree.py +++ b/src/pynxtools/dataconverter/nexus_tree.py @@ -48,6 +48,7 @@ "NX_BOOLEAN", "NX_CCOMPLEX", "NX_CHAR", + "NX_CHAR_OR_NUMBER", "NX_COMPLEX", "NX_DATE_TIME", "NX_FLOAT", From 19eea47674b336377cdc1bbfd35b98ee9cbb91eb Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 11 Mar 2025 18:29:03 +0100 Subject: [PATCH 11/52] properly build inheritance and siblings --- src/pynxtools/dataconverter/nexus_tree.py | 89 ++++++++++++++++------- 1 file changed, 64 insertions(+), 25 deletions(-) diff --git a/src/pynxtools/dataconverter/nexus_tree.py b/src/pynxtools/dataconverter/nexus_tree.py index ddf343b21..0a72d34bf 100644 --- a/src/pynxtools/dataconverter/nexus_tree.py +++ b/src/pynxtools/dataconverter/nexus_tree.py @@ -35,13 +35,16 @@ from anytree.node.nodemixin import NodeMixin from pynxtools.dataconverter.helpers import ( - contains_uppercase, get_all_parents_for, get_nxdl_root_and_path, + is_variadic, is_appdef, remove_namespace_from_tag, ) -from pynxtools.definitions.dev_tools.utils.nxdl_utils import get_nx_namefit +from pynxtools.definitions.dev_tools.utils.nxdl_utils import ( + get_nx_namefit, + is_name_type, +) NexusType = Literal[ "NX_BINARY", @@ -193,7 +196,7 @@ def __init__( self.type = type self.name_type = name_type self.optionality = optionality - self.variadic = self._is_variadic(self.name, self.name_type) + self.variadic = is_variadic(self.name, self.name_type) if variadic is not None: self.variadic = variadic if inheritance is not None: @@ -204,14 +207,6 @@ def __init__( self.is_a = [] self.parent_of = [] - def _is_variadic(self, name: str, name_type: str) -> bool: - """ - Determine if a name is variadic based on its nameType. - """ - if name: - return False if name_type == "specified" else True - return True - def _construct_inheritance_chain_from_parent(self): """ Builds the inheritance chain of the current node based on the parent node. @@ -475,6 +470,7 @@ def _build_inheritance_chain(self, xml_elem: ET._Element) -> List[ET._Element]: This represents the direct field or group inside the specific xml file. """ name = xml_elem.attrib.get("name") + inheritance_chain = [xml_elem] inheritance = iter(self.inheritance) for elem in inheritance: @@ -498,18 +494,28 @@ def _build_inheritance_chain(self, xml_elem: ET._Element) -> List[ET._Element]: best_group = None best_score = -1 for group in groups: - if name in group.attrib and not contains_uppercase( - group.attrib["name"] - ): - continue group_name = ( group.attrib.get("name") if "name" in group.attrib else group.attrib["type"][2:].upper() ) - score = get_nx_namefit(name, group_name) - if get_nx_namefit(name, group_name) >= best_score: + if "name" in group.attrib: + group_name_type = group.attrib.get("nameType", "specified") + + else: + group_name_type = group.attrib.get("nameType", "any") + + if not is_variadic(group_name, group_name_type): + continue + + group_name_any = is_name_type(group, "any") + group_name_partial = is_name_type(group, "partial") + + score = get_nx_namefit( + name, group_name, group_name_any, group_name_partial + ) + if score >= best_score: best_group = group best_score = score @@ -557,7 +563,7 @@ def add_node_from(self, xml_elem: ET._Element) -> Optional["NexusNode"]: name = xml_elem.attrib.get("name") if not name: name = xml_elem.attrib["type"][2:].upper() - name_type = None + name_type = "any" inheritance_chain = self._build_inheritance_chain(xml_elem) current_elem = NexusGroup( @@ -669,6 +675,7 @@ def _check_sibling_namefit(self): for elem in self.inheritance[1:]: parent = elem.getparent() + if parent is None: continue siblings = parent.findall( @@ -676,15 +683,43 @@ def _check_sibling_namefit(self): ) for sibling in siblings: - sibling_name = sibling.attrib.get( - "name", sibling.attrib["type"][2:].upper() + sibling_name = ( + sibling.attrib.get("name") + if "name" in sibling.attrib + else sibling.attrib["type"][2:].upper() ) - if sibling_name == self.name or not contains_uppercase(sibling_name): + + if "name" in sibling.attrib: + sibling_name_type = sibling.attrib.get("nameType", "specified") + else: + sibling_name_type = sibling.attrib.get("nameType", "any") + + if not is_variadic(sibling_name, sibling_name_type): continue - if get_nx_namefit(self.name, sibling_name) < 0: + + # sibling_name = sibling.attrib.get("name") + # if sibling_name: + # sibling_name_type = sibling.attrib.get("nameType", "specified") + # else: + # sibling_name = sibling.attrib["type"][2:].upper() + # sibling_name_type = "any" + + # if sibling_name == self.name or not is_variadic(, sibling_name_type): + # continue + + sibling_name_any = is_name_type(sibling, "any") + sibling_name_partial = is_name_type(sibling, "partial") + + if ( + get_nx_namefit( + self.name, sibling_name, sibling_name_any, sibling_name_partial + ) + < 0 + ): continue sibling_node = self.parent.get_child_for(sibling) + if sibling_node is None: sibling_node = self.parent.add_node_from(sibling) self.is_a.append(sibling_node) @@ -742,9 +777,13 @@ def __init__(self, nx_class: str, **data) -> None: self._check_sibling_namefit() def __repr__(self) -> str: - return ( - f"{self.nx_class[2:].upper()}[{self.name.lower()}] ({self.optionality[:3]})" - ) + inh_str = "\n ".join(str(parent.attrib) for parent in self.inheritance) + sib_str = "\n ".join(str(sibling) for sibling in self.is_a) + + inh_part = f"\n inh:\n {inh_str}" if inh_str else "" + sib_part = f"\n sib: {sib_str}" if sib_str else "" + + return f"{self.nx_class[2:].upper()}[{self.name.lower()}] ({self.optionality}, nameType: {self.name_type}{inh_part}{sib_part})" class NexusEntity(NexusNode): From 3032895357c5a7f9b528c8513274c41833100508 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 11 Mar 2025 19:37:40 +0100 Subject: [PATCH 12/52] extends NXobject in NeXusTree --- src/pynxtools/dataconverter/helpers.py | 11 +++++++++- src/pynxtools/dataconverter/nexus_tree.py | 25 +++++++++-------------- src/pynxtools/dataconverter/validation.py | 8 +++++--- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/pynxtools/dataconverter/helpers.py b/src/pynxtools/dataconverter/helpers.py index 4a1dfbc6c..c12208d08 100644 --- a/src/pynxtools/dataconverter/helpers.py +++ b/src/pynxtools/dataconverter/helpers.py @@ -267,7 +267,7 @@ def get_all_parents_for(xml_elem: ET._Element) -> List[ET._Element]: root = get_appdef_root(xml_elem) inheritance_chain = [] extends = root.get("extends") - while extends is not None and extends != "NXobject": + while extends is not None: parent_xml_root, _ = get_nxdl_root_and_path(extends) extends = parent_xml_root.get("extends") inheritance_chain.append(parent_xml_root) @@ -490,6 +490,15 @@ def contains_uppercase(field_name: Optional[str]) -> bool: return any(char.isupper() for char in field_name) +def is_variadic(name: str, name_type: str) -> bool: + """ + Determine if a name is variadic based on its nameType. + """ + if name: + return False if name_type == "specified" else True + return True + + def convert_nexus_to_suggested_name(nexus_name): """Helper function to suggest a name for a group from its NeXus class.""" if contains_uppercase(nexus_name): diff --git a/src/pynxtools/dataconverter/nexus_tree.py b/src/pynxtools/dataconverter/nexus_tree.py index 0a72d34bf..48178a96c 100644 --- a/src/pynxtools/dataconverter/nexus_tree.py +++ b/src/pynxtools/dataconverter/nexus_tree.py @@ -697,16 +697,6 @@ def _check_sibling_namefit(self): if not is_variadic(sibling_name, sibling_name_type): continue - # sibling_name = sibling.attrib.get("name") - # if sibling_name: - # sibling_name_type = sibling.attrib.get("nameType", "specified") - # else: - # sibling_name = sibling.attrib["type"][2:].upper() - # sibling_name_type = "any" - - # if sibling_name == self.name or not is_variadic(, sibling_name_type): - # continue - sibling_name_any = is_name_type(sibling, "any") sibling_name_partial = is_name_type(sibling, "partial") @@ -777,13 +767,18 @@ def __init__(self, nx_class: str, **data) -> None: self._check_sibling_namefit() def __repr__(self) -> str: - inh_str = "\n ".join(str(parent.attrib) for parent in self.inheritance) - sib_str = "\n ".join(str(sibling) for sibling in self.is_a) + if self.type == "attribute": + return f"@{self.name} ({self.optionality[:3]})" + return f"{self.name} ({self.optionality[:3]})" + + # def __repr__(self) -> str: + # inh_str = "\n ".join(str(parent.attrib) for parent in self.inheritance) + # sib_str = "\n ".join(str(sibling) for sibling in self.is_a) - inh_part = f"\n inh:\n {inh_str}" if inh_str else "" - sib_part = f"\n sib: {sib_str}" if sib_str else "" + # inh_part = f"\n inh:\n {inh_str}" if inh_str else "" + # sib_part = f"\n sib: {sib_str}" if sib_str else "" - return f"{self.nx_class[2:].upper()}[{self.name.lower()}] ({self.optionality}, nameType: {self.name_type}{inh_part}{sib_part})" + # return f"{self.nx_class[2:].upper()}[{self.name.lower()}] ({self.optionality}, nameType: {self.name_type}{inh_part}{sib_part})" class NexusEntity(NexusNode): diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index f255c7e2a..176abede6 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -156,8 +156,8 @@ def best_namefit_of(name: str, keys: Iterable[str], name_type: str) -> Optional[ if nx_name is not None and nx_name in keys: return nx_name - name_any = True if name_type == "any" else False - name_partial = True if name_type == "partial" else False + name_any = name_type == "any" + name_partial = name_type == "partial" best_match, score = max( map(lambda x: (x, get_nx_namefit(name2fit, x, name_any, name_partial)), keys), @@ -522,11 +522,13 @@ def handle_unknown_type(node: NexusNode, keys: Mapping[str, Any], prev_path: str def is_documented(key: str, node: NexusNode) -> bool: if mapping.get(key) is None: - # This value is not really set. Skip checking it's documentation. + # This value is not really set. Skip checking its documentation. return True for name in key[1:].replace("@", "").split("/"): children = node.get_all_direct_children_names() + if name == "data": + print(name, children, node.name_type, node.__dict__) best_name = best_namefit_of(name, children, node.name_type) if best_name is None: return False From 94193cac9bab298dfe591b04ea1835fe2bb7ab3a Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 11 Mar 2025 19:44:12 +0100 Subject: [PATCH 13/52] update defs to feature branch for dev_tools --- .gitmodules | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 71907ead7..4170aa516 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "src/pynxtools/definitions"] path = src/pynxtools/definitions - url = https://github.com/FAIRmat-NFDI/nexus_definitions.git \ No newline at end of file + url = https://github.com/FAIRmat-NFDI/nexus_definitions.git + branch = namefitting-tests \ No newline at end of file From c4e57f9a29903573de957ab53f9a0721b8b580b6 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 11 Mar 2025 20:31:52 +0100 Subject: [PATCH 14/52] implement test for open enum --- src/pynxtools/dataconverter/validation.py | 8 +++----- tests/dataconverter/test_helpers.py | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index 176abede6..649b200da 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -209,8 +209,8 @@ def get_variations_of(node: NexusNode, keys: Mapping[str, Any]) -> List[str]: name2fit = name2fit[1:] if name2fit.startswith("@") else name2fit if nx_name is not None and nx_name != node.name: continue - name_any = True if node.name_type in ("any", None) else False - name_partial = True if node.name_type == "partial" else False + name_any = node.name_type == "any" + name_partial = node.name_type == "partial" if ( get_nx_namefit(name2fit, node.name, name_any, name_partial) >= 0 @@ -362,7 +362,7 @@ def handle_group(node: NexusGroup, keys: Mapping[str, Any], prev_path: str): ValidationProblem.ExpectedGroup, None, ) - return + continue if node.nx_class == "NXdata": handle_nxdata(node, keys[variant], prev_path=f"{prev_path}/{variant}") else: @@ -527,8 +527,6 @@ def is_documented(key: str, node: NexusNode) -> bool: for name in key[1:].replace("@", "").split("/"): children = node.get_all_direct_children_names() - if name == "data": - print(name, children, node.name_type, node.__dict__) best_name = best_namefit_of(name, children, node.name_type) if best_name is None: return False diff --git a/tests/dataconverter/test_helpers.py b/tests/dataconverter/test_helpers.py index 0ef64ea0d..a11a35197 100644 --- a/tests/dataconverter/test_helpers.py +++ b/tests/dataconverter/test_helpers.py @@ -438,6 +438,15 @@ def fixture_filled_test_data(template, tmp_path): ), id="wrong-enum-choice", ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/type2", + "a very different type", + ), + "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type2 does not match with the enumerated items from the open enumeration: ['1st type open', '2nd type open'].", + id="open-enum-with-new-item", + ), pytest.param( set_to_none_in_dict( TEMPLATE, "/ENTRY[my_entry]/optional_parent/required_child", "optional" @@ -523,7 +532,11 @@ def fixture_filled_test_data(template, tmp_path): ) def test_validate_data_dict(caplog, data_dict, error_message, request): """Unit test for the data validation routine.""" - if request.node.callspec.id in ( + if request.node.callspec.id in ("open-enum-with-new-item",): + with caplog.at_level(logging.INFO): + assert not validate_dict_against("NXtest", data_dict)[0] + assert error_message in caplog.text + elif request.node.callspec.id in ( "valid-data-dict", "lists", "empty-optional-field", @@ -553,7 +566,6 @@ def test_validate_data_dict(caplog, data_dict, error_message, request): else: with caplog.at_level(logging.WARNING): assert not validate_dict_against("NXtest", data_dict)[0] - assert error_message in caplog.text From dead37b2bc5d97ee37ab0eae4f2c6035b7cb2fce Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 11 Mar 2025 21:28:56 +0100 Subject: [PATCH 15/52] reorganize open enum tests in validatation --- tests/dataconverter/test_validation.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index e213857f4..d8472cfa7 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -147,7 +147,7 @@ def test_validation_shows_warning(caplog, data_dict, error_message): @pytest.mark.parametrize( - "data_dict, message, expected_fail", + "data_dict, message, log_level", [ pytest.param( alter_dict( @@ -157,7 +157,7 @@ def test_validation_shows_warning(caplog, data_dict, error_message): get_data_dict(), ), "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type should be on of the following strings: ['1st type', '2nd type', '3rd type', '4th type']", - True, + logging.WARNING, id="closed-enum-with-new-item", ), pytest.param( @@ -168,12 +168,12 @@ def test_validation_shows_warning(caplog, data_dict, error_message): get_data_dict(), ), "The value at /ENTRY[my_entry]/NXODD_name[nxodd_name]/type2 does not match with the enumerated items from the open enumeration: ['1st type open', '2nd type open'].", - False, + logging.INFO, id="open-enum-with-new-item", ), ], ) -def test_validation_enumeration(caplog, data_dict, message, expected_fail): - with caplog.at_level(logging.WARNING): - assert validate_dict_against("NXtest", data_dict)[0] == (not expected_fail) +def test_validation_enumeration(caplog, data_dict, message, log_level): + with caplog.at_level(log_level): + assert not validate_dict_against("NXtest", data_dict)[0] assert message in caplog.text From 8ecad8138845669de1beeadc368af1f59d73bac7 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 11 Mar 2025 21:52:23 +0100 Subject: [PATCH 16/52] adjust namefitting for is_documented --- src/pynxtools/dataconverter/nexus_tree.py | 7 ++- src/pynxtools/dataconverter/validation.py | 43 ++++++++++++------- .../readers/example/testdata.json | 1 + 3 files changed, 33 insertions(+), 18 deletions(-) diff --git a/src/pynxtools/dataconverter/nexus_tree.py b/src/pynxtools/dataconverter/nexus_tree.py index 48178a96c..fbe26e5a7 100644 --- a/src/pynxtools/dataconverter/nexus_tree.py +++ b/src/pynxtools/dataconverter/nexus_tree.py @@ -337,6 +337,7 @@ def get_all_direct_children_names( The inheritance depth up to which get children names. `depth=1` will return only the children of the current node. `depth=None` will return all children names of all parents. + `depth=-1` will return all children names of all parents, except for NXobject. Defaults to None. only_appdef (bool, optional): Only considers appdef nodes as children. @@ -348,8 +349,10 @@ def get_all_direct_children_names( Returns: Set[str]: A set of children names. """ - if depth is not None and (not isinstance(depth, int) or depth < 0): - raise ValueError("Depth must be a positive integer or None") + if depth is not None and ( + not isinstance(depth, int) or (depth < 0 and depth != -1) + ): + raise ValueError("Depth must be a positive integer, -1 or None") tag_type = "" if node_type == "group" and nx_class is not None: diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index 649b200da..e60798a38 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -134,7 +134,7 @@ def split_class_and_name_of(name: str) -> Tuple[Optional[str], str]: ), f"{name_match.group(2)}{'' if prefix is None else prefix}" -def best_namefit_of(name: str, keys: Iterable[str], name_type: str) -> Optional[str]: +def best_namefit_of(name: str, nodes: Iterable[NexusNode]) -> Optional[str]: """ Get the best namefit of `name` in `keys`. @@ -146,24 +146,30 @@ def best_namefit_of(name: str, keys: Iterable[str], name_type: str) -> Optional[ Returns: Optional[str]: The best fitting key. None if no fit was found. """ - if not keys: + if not nodes: return None nx_name, name2fit = split_class_and_name_of(name) - if name2fit in keys: - return name2fit - if nx_name is not None and nx_name in keys: - return nx_name + best_match = 0 + best_score = -1 - name_any = name_type == "any" - name_partial = name_type == "partial" + for node in nodes: + if name2fit == node.name: + return name2fit + if nx_name is not None and nx_name == node.name: + return nx_name - best_match, score = max( - map(lambda x: (x, get_nx_namefit(name2fit, x, name_any, name_partial)), keys), - key=lambda x: x[1], - ) - if score < 0: + name_any = node.name_type == "any" + name_partial = node.name_type == "partial" + + score = get_nx_namefit(name2fit, node.name, name_any, name_partial) + + if score > best_score: + best_score = score + best_match = node.name + + if best_score < 0: return None return best_match @@ -526,8 +532,13 @@ def is_documented(key: str, node: NexusNode) -> bool: return True for name in key[1:].replace("@", "").split("/"): - children = node.get_all_direct_children_names() - best_name = best_namefit_of(name, children, node.name_type) + children_to_check = [ + node.search_add_child_for(child) + for child in node.get_all_direct_children_names(depth=-1) + ] + best_name = best_namefit_of(name, children_to_check) + if "float_value_no_attr" in name: + print(name, children_to_check, best_name) if best_name is None: return False @@ -673,7 +684,7 @@ def check_type_with_tree( if (next_child_class is not None) or (next_child_name is not None): output = None for child in node.children: - # regexs to separarte the class and the name from full name of the child + # regexs to separate the class and the name from full name of the child child_class_from_node = re.sub( r"(\@.*)*(\[.*?\])*(\(.*?\))*([a-z]\_)*(\_[a-z])*[a-z]*\s*", "", diff --git a/tests/data/dataconverter/readers/example/testdata.json b/tests/data/dataconverter/readers/example/testdata.json index 21deb40c3..9a5dac737 100644 --- a/tests/data/dataconverter/readers/example/testdata.json +++ b/tests/data/dataconverter/readers/example/testdata.json @@ -13,6 +13,7 @@ "definition_version": "0.0.1", "program_name": "Nexus Parser", "type": "2nd type", + "type2": "2nd type", "date_value": "2022-01-22T12:14:12.05018+00:00", "date_value_units": "", "required_child": 1, From df9f59acff684f55c86938a6ef13acbfc4a1971f Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 11 Mar 2025 22:32:48 +0100 Subject: [PATCH 17/52] test for identifier in nested NXdata --- src/pynxtools/dataconverter/nexus_tree.py | 7 ++----- src/pynxtools/dataconverter/validation.py | 8 +++++--- tests/dataconverter/test_validation.py | 2 ++ 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/pynxtools/dataconverter/nexus_tree.py b/src/pynxtools/dataconverter/nexus_tree.py index fbe26e5a7..48178a96c 100644 --- a/src/pynxtools/dataconverter/nexus_tree.py +++ b/src/pynxtools/dataconverter/nexus_tree.py @@ -337,7 +337,6 @@ def get_all_direct_children_names( The inheritance depth up to which get children names. `depth=1` will return only the children of the current node. `depth=None` will return all children names of all parents. - `depth=-1` will return all children names of all parents, except for NXobject. Defaults to None. only_appdef (bool, optional): Only considers appdef nodes as children. @@ -349,10 +348,8 @@ def get_all_direct_children_names( Returns: Set[str]: A set of children names. """ - if depth is not None and ( - not isinstance(depth, int) or (depth < 0 and depth != -1) - ): - raise ValueError("Depth must be a positive integer, -1 or None") + if depth is not None and (not isinstance(depth, int) or depth < 0): + raise ValueError("Depth must be a positive integer or None") tag_type = "" if node_type == "group" and nx_class is not None: diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index e60798a38..925d7aaad 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -534,11 +534,13 @@ def is_documented(key: str, node: NexusNode) -> bool: for name in key[1:].replace("@", "").split("/"): children_to_check = [ node.search_add_child_for(child) - for child in node.get_all_direct_children_names(depth=-1) + for child in node.get_all_direct_children_names() ] best_name = best_namefit_of(name, children_to_check) - if "float_value_no_attr" in name: - print(name, children_to_check, best_name) + + # if "float_value_no_attr" in name or "identifier" in name: + # print(name, best_name) + if best_name is None: return False diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index d8472cfa7..c84247f40 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -66,6 +66,8 @@ def get_data_dict(): "/ENTRY[my_entry]/required_group/description": "An example description", "/ENTRY[my_entry]/required_group2/description": "An example description", "/ENTRY[my_entry]/optional_parent/req_group_in_opt_group/data": 1, + "/ENTRY[my_entry]/optional_parent/req_group_in_opt_group/identifier": "my_identifier", + "/ENTRY[my_entry]/optional_parent/req_group_in_opt_group/identifier1": "my_identifier1", "/@default": "Some NXroot attribute", } From c5087f89564d17c476800078882ca4818ad2d2d9 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 11 Mar 2025 23:02:12 +0100 Subject: [PATCH 18/52] remove improper test value from NXdata --- src/pynxtools/dataconverter/validation.py | 5 ++--- tests/dataconverter/test_validation.py | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index 925d7aaad..3a00795f7 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -146,6 +146,7 @@ def best_namefit_of(name: str, nodes: Iterable[NexusNode]) -> Optional[str]: Returns: Optional[str]: The best fitting key. None if no fit was found. """ + PRINT = False # True if "float_value_no_attr" in name else False if not nodes: return None @@ -538,9 +539,6 @@ def is_documented(key: str, node: NexusNode) -> bool: ] best_name = best_namefit_of(name, children_to_check) - # if "float_value_no_attr" in name or "identifier" in name: - # print(name, best_name) - if best_name is None: return False @@ -578,6 +576,7 @@ def recurse_tree( keys = _follow_link(keys, prev_path) if keys is None: return + handling_map.get(child.type, handle_unknown_type)(child, keys, prev_path) def check_attributes_of_nonexisting_field( diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index c84247f40..bf6abbf2f 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -28,7 +28,6 @@ def get_data_dict(): return { "/ENTRY[my_entry]/optional_parent/required_child": 1, "/ENTRY[my_entry]/optional_parent/optional_child": 1, - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value_no_attr": 2.0, "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value": 2.0, "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value/@units": "nm", "/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value": True, @@ -94,7 +93,7 @@ def alter_dict(new_values: Dict[str, Any], data_dict: Dict[str, Any]) -> Dict[st pytest.param(get_data_dict(), id="valid-unaltered-data-dict"), pytest.param( remove_from_dict( - "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value_no_attr", + "/ENTRY[my_entry]/NXODD_name[nxodd_name]/float_value", get_data_dict(), ), id="removed-optional-value", From 06405eb9fa5f6a34ed7e94c4b53d637e8b542c96 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Wed, 12 Mar 2025 10:05:19 +0100 Subject: [PATCH 19/52] change back to fairmat branch for definitions --- .gitmodules | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 4170aa516..71907ead7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,3 @@ [submodule "src/pynxtools/definitions"] path = src/pynxtools/definitions - url = https://github.com/FAIRmat-NFDI/nexus_definitions.git - branch = namefitting-tests \ No newline at end of file + url = https://github.com/FAIRmat-NFDI/nexus_definitions.git \ No newline at end of file From 6a3ae123819595fe9aeb06cd170dc8414738001e Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Wed, 12 Mar 2025 10:33:32 +0100 Subject: [PATCH 20/52] use new NXroot attributs --- src/pynxtools/dataconverter/helpers.py | 2 +- tests/dataconverter/test_helpers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pynxtools/dataconverter/helpers.py b/src/pynxtools/dataconverter/helpers.py index c12208d08..bd5eef0ba 100644 --- a/src/pynxtools/dataconverter/helpers.py +++ b/src/pynxtools/dataconverter/helpers.py @@ -917,7 +917,7 @@ def update_and_warn(key: str, value: str): "https://github.com/FAIRmat-NFDI/nexus_definitions/" f"blob/{get_nexus_version_hash()}", ) - update_and_warn("/@NeXus_version", get_nexus_version()) + update_and_warn("/@NeXus_release", get_nexus_version()) update_and_warn("/@HDF5_version", ".".join(map(str, h5py.h5.get_libversion()))) update_and_warn("/@h5py_version", h5py.__version__) diff --git a/tests/dataconverter/test_helpers.py b/tests/dataconverter/test_helpers.py index a11a35197..1c0553237 100644 --- a/tests/dataconverter/test_helpers.py +++ b/tests/dataconverter/test_helpers.py @@ -619,7 +619,7 @@ def test_writing_of_root_attributes(caplog): assert "/@file_time" in keys_added assert "/@file_update_time" in keys_added assert "/@NeXus_repository" in keys_added - assert "/@NeXus_version" in keys_added + assert "/@NeXus_release" in keys_added assert "/@HDF5_version" in keys_added assert "/@h5py_version" in keys_added assert "/ENTRY[entry]/definition" in keys_added From 0dfa370008e3731a9d44e3ed7c9bddfb2fa22fb9 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Wed, 12 Mar 2025 15:08:15 +0100 Subject: [PATCH 21/52] support complex units like eV/mm in nomad schema generation --- pyproject.toml | 1 + src/pynxtools/nomad/schema.py | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 622225b3b..88bb7f6f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "importlib-metadata", "lxml>=4.9.1", "anytree", + "pint==0.17", ] [project.urls] diff --git a/src/pynxtools/nomad/schema.py b/src/pynxtools/nomad/schema.py index 767761219..c30811b85 100644 --- a/src/pynxtools/nomad/schema.py +++ b/src/pynxtools/nomad/schema.py @@ -39,6 +39,7 @@ from nomad.normalizing.common import nomad_atoms_from_ase_atoms from nomad.normalizing.topology import add_system, add_system_info from scipy.spatial import cKDTree +import pint try: from nomad import utils @@ -690,11 +691,19 @@ def __create_field(xml_node: ET.Element, container: Section) -> Quantity: # dimensionality nx_dimensionality = xml_attrs.get("units", None) if nx_dimensionality: - if nx_dimensionality not in NXUnitSet.mapping: - raise NotImplementedError( - f"Unit {nx_dimensionality} is not supported for {name}." - ) - dimensionality = NXUnitSet.mapping[nx_dimensionality] + dimensionality = NXUnitSet.mapping.get(nx_dimensionality) + if not dimensionality and nx_dimensionality != "NX_ANY": + nx_dimensionality = "asdjkalsdbasjdk" + try: + # nx_dimensionality = "some_chasdasdkasdn m" + from nomad.units import ureg + + quantity = 1 * ureg(nx_dimensionality) + dimensionality = quantity.dimensionality + except pint.errors.UndefinedUnitError as err: + raise NotImplementedError( + f"Unit {nx_dimensionality} is not supported for {name}." + ) from err else: dimensionality = None From c3f4428818e9013d4f91d401c393f5adbea9b41a Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Wed, 12 Mar 2025 15:14:52 +0100 Subject: [PATCH 22/52] mypy fix --- src/pynxtools/dataconverter/validation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index 3a00795f7..f44754b21 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -152,7 +152,7 @@ def best_namefit_of(name: str, nodes: Iterable[NexusNode]) -> Optional[str]: nx_name, name2fit = split_class_and_name_of(name) - best_match = 0 + best_match = None best_score = -1 for node in nodes: From b186762331abe2ab7f9eae0c8f424a5de2288f4c Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Wed, 12 Mar 2025 15:16:31 +0100 Subject: [PATCH 23/52] remove print statements --- src/pynxtools/dataconverter/validation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index f44754b21..6948271a6 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -146,7 +146,6 @@ def best_namefit_of(name: str, nodes: Iterable[NexusNode]) -> Optional[str]: Returns: Optional[str]: The best fitting key. None if no fit was found. """ - PRINT = False # True if "float_value_no_attr" in name else False if not nodes: return None From 3b0c95628929c2f3c5455d3610dff12581ccc4b0 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Wed, 12 Mar 2025 15:53:23 +0100 Subject: [PATCH 24/52] adjust docstrings --- src/pynxtools/dataconverter/validation.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index 6948271a6..819108bb8 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -141,7 +141,6 @@ def best_namefit_of(name: str, nodes: Iterable[NexusNode]) -> Optional[str]: Args: name (str): The name to fit against the keys. keys (Iterable[str]): The keys to fit `name` against. - name_type (str): nameType of the concept being fitted Returns: Optional[str]: The best fitting key. None if no fit was found. From 61e30ea29c00216f34ad72c1346ff2bcff548987 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Fri, 14 Mar 2025 16:39:20 +0100 Subject: [PATCH 25/52] update definitions once more --- src/pynxtools/definitions | 2 +- src/pynxtools/nexus-version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pynxtools/definitions b/src/pynxtools/definitions index 3f66054d6..db54d2a03 160000 --- a/src/pynxtools/definitions +++ b/src/pynxtools/definitions @@ -1 +1 @@ -Subproject commit 3f66054d6b1651617fdfbb24d4b2bfa33f75de66 +Subproject commit db54d2a0394c91898092b67ec71508e0f805ec57 diff --git a/src/pynxtools/nexus-version.txt b/src/pynxtools/nexus-version.txt index 3f2a82f53..48133eb9b 100644 --- a/src/pynxtools/nexus-version.txt +++ b/src/pynxtools/nexus-version.txt @@ -1 +1 @@ -v2024.02-1753-g3f66054d \ No newline at end of file +v2024.02-1885-gdb54d2a0 \ No newline at end of file From 098fe0eb7adcef403f75616fee5953b56c52f5d2 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Mon, 17 Mar 2025 13:21:55 +0100 Subject: [PATCH 26/52] remove NXtest2, check NXtest more thoroughly --- tests/data/nexus/NXtest2.nxdl.xml | 455 ------------------------------ tests/nexus/test_nexus.py | 82 ++---- 2 files changed, 23 insertions(+), 514 deletions(-) delete mode 100644 tests/data/nexus/NXtest2.nxdl.xml diff --git a/tests/data/nexus/NXtest2.nxdl.xml b/tests/data/nexus/NXtest2.nxdl.xml deleted file mode 100644 index 7b33b2165..000000000 --- a/tests/data/nexus/NXtest2.nxdl.xml +++ /dev/null @@ -1,455 +0,0 @@ - - - - - - - Characterization of a sample during a session on an electron microscope. - - - - - - - - Metadata and numerical data of the microscope and the lab in which it stands. - - - - - - Given name of the microscope at the hosting institution. This is an alias. - Examples could be NionHermes, Titan, JEOL, Gemini, etc. - - - - - Location of the lab or place where the instrument is installed. - Using GEOREF is preferred. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - If the lens is described at least one of the fields - voltage, current, or value should be defined. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Description of the type of the detector. - - Electron microscopes have typically multiple detectors. - Different technologies are in use like CCD, scintillator, - direct electron, CMOS, or image plate to name but a few. - - - - Instrument-specific alias/name - - - - - - - - - - - - - - - - - - - - - - - - - - - - - A container for storing a set of NXevent_data_em instances. - - - - - \ No newline at end of file diff --git a/tests/nexus/test_nexus.py b/tests/nexus/test_nexus.py index 2069f4dd1..4b14b9753 100644 --- a/tests/nexus/test_nexus.py +++ b/tests/nexus/test_nexus.py @@ -213,77 +213,41 @@ def test_get_node_at_nxdl_path(): """Test to verify if we receive the right XML element for a given NXDL path""" local_dir = os.path.abspath(os.path.dirname(__file__)) nxdl_file_path = os.path.join(local_dir, "../../src/pynxtools/data/NXtest.nxdl.xml") + elem = ET.parse(nxdl_file_path).getroot() + node = get_node_at_nxdl_path("/ENTRY/NXODD_name", elem=elem) assert node.attrib["type"] == "NXdata" assert node.attrib["name"] == "NXODD_name" + node = get_node_at_nxdl_path("/ENTRY/NXODD_name/anamethatRENAMES", elem=elem) + assert node.attrib["type"] == "NX_INT" + assert node.attrib["name"] == "anamethatRENAMES" + assert node.attrib["nameType"] == "partial" + assert node.attrib["units"] == "NX_UNITLESS" + node = get_node_at_nxdl_path("/ENTRY/NXODD_name/float_value", elem=elem) assert node.attrib["type"] == "NX_FLOAT" assert node.attrib["name"] == "float_value" + assert not node.attrib.get("nameType") node = get_node_at_nxdl_path("/ENTRY/NXODD_name/AXISNAME/long_name", elem=elem) assert node.attrib["name"] == "long_name" - nxdl_file_path = os.path.join(local_dir, "../data/nexus/NXtest2.nxdl.xml") - elem = ET.parse(nxdl_file_path).getroot() - node = get_node_at_nxdl_path( - "/ENTRY/measurement/EVENT_DATA_EM/USER/affiliation", elem=elem - ) - assert node.attrib["name"] == "affiliation" - - node = get_node_at_nxdl_path("/ENTRY/measurement", elem=elem) - assert node.attrib["type"] == "NXevent_data_em_set" - - node = get_node_at_nxdl_path( - "/ENTRY/measurement/EVENT_DATA_EM/SPECTRUM_SET/stack_3d", elem=elem - ) - assert node.attrib["type"] == "NXdata" - - node = get_node_at_nxdl_path( - "/ENTRY/measurement/EVENT_DATA_EM/SPECTRUM_SET/stack_3d/intensity", elem=elem - ) - assert node.attrib["type"] == "NX_NUMBER" + node = get_node_at_nxdl_path("/ENTRY/NXODD_name/group_attribute", elem=elem) + assert node.attrib["name"] == "group_attribute" - node = get_node_at_nxdl_path( - "/ENTRY/measurement/EVENT_DATA_EM/SPECTRUM_SET/stack_3d/AXISNAME_indices", - elem=elem, - ) - assert node.attrib["name"] == "AXISNAME_indices" - - node = get_node_at_nxdl_path("/ENTRY/COORDINATE_SYSTEM_SET", elem=elem) - assert node.attrib["type"] == "NXcoordinate_system_set" + node = get_node_at_nxdl_path("/ENTRY/optional_parent", elem=elem) + assert node.attrib["name"] == "optional_parent" + assert node.attrib["optional"] == "true" - node = get_node_at_nxdl_path( - "/ENTRY/COORDINATE_SYSTEM_SET/TRANSFORMATIONS", elem=elem - ) - assert node.attrib["type"] == "NXtransformations" + node = get_node_at_nxdl_path("/ENTRY/optional_parent/required_child", elem=elem) + assert node.attrib["name"] == "required_child" + assert node.attrib["type"] == "NX_INT" + assert node.attrib["required"] == "true" - node = get_node_at_nxdl_path( - "/ENTRY/COORDINATE_SYSTEM_SET/TRANSFORMATIONS/AXISNAME", elem=elem - ) - assert node.attrib["type"] == "NX_NUMBER" - - node = get_node_at_nxdl_path( - "/ENTRY/COORDINATE_SYSTEM_SET/TRANSFORMATIONS/AXISNAME/transformation_type", - elem=elem, - ) - assert node.attrib["name"] == "transformation_type" - - nxdl_file_path = os.path.join( - local_dir, - "../../src/pynxtools/definitions/contributed_definitions/NXiv_temp.nxdl.xml", - ) - elem = ET.parse(nxdl_file_path).getroot() - node = get_node_at_nxdl_path( - "/ENTRY/INSTRUMENT/ENVIRONMENT/voltage_controller", elem=elem - ) - assert node.attrib["name"] == "voltage_controller" - - node = get_node_at_nxdl_path( - "/ENTRY/INSTRUMENT/ENVIRONMENT/voltage_controller/calibration_time", elem=elem - ) - assert node.attrib["name"] == "calibration_time" + node = get_node_at_nxdl_path("/ENTRY/USER/affiliation", elem=elem) + assert node.attrib["name"] == "affiliation" def test_get_inherited_nodes(): @@ -297,18 +261,18 @@ def test_get_inherited_nodes(): (_, _, elist) = get_inherited_nodes( nxdl_path="/ENTRY/INSTRUMENT/ENVIRONMENT", elem=elem ) - assert len(elist) == 3 + assert len(elist) == 4 (_, _, elist) = get_inherited_nodes( nxdl_path="/ENTRY/INSTRUMENT/ENVIRONMENT/voltage_controller", elem=elem ) - assert len(elist) == 4 + assert len(elist) == 6 (_, _, elist) = get_inherited_nodes( nxdl_path="/ENTRY/INSTRUMENT/ENVIRONMENT/voltage_controller", nx_name="NXiv_temp", ) - assert len(elist) == 4 + assert len(elist) == 6 def test_c_option(tmp_path): From 76d32cdfd90f1a432cc37f01ba53f234ac917b45 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Mon, 17 Mar 2025 13:41:29 +0100 Subject: [PATCH 27/52] remove print statement --- src/pynxtools/nomad/schema.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pynxtools/nomad/schema.py b/src/pynxtools/nomad/schema.py index c30811b85..46a97a396 100644 --- a/src/pynxtools/nomad/schema.py +++ b/src/pynxtools/nomad/schema.py @@ -693,7 +693,6 @@ def __create_field(xml_node: ET.Element, container: Section) -> Quantity: if nx_dimensionality: dimensionality = NXUnitSet.mapping.get(nx_dimensionality) if not dimensionality and nx_dimensionality != "NX_ANY": - nx_dimensionality = "asdjkalsdbasjdk" try: # nx_dimensionality = "some_chasdasdkasdn m" from nomad.units import ureg From 4572927d4c00480aaac391446e488f6845fcb822 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Mon, 17 Mar 2025 13:47:12 +0100 Subject: [PATCH 28/52] update ref nexus log --- tests/data/nexus/Ref_nexus_test.log | 1170 +++++++++++++++------------ 1 file changed, 664 insertions(+), 506 deletions(-) diff --git a/tests/data/nexus/Ref_nexus_test.log b/tests/data/nexus/Ref_nexus_test.log index 54541649f..7f7279708 100644 --- a/tests/data/nexus/Ref_nexus_test.log +++ b/tests/data/nexus/Ref_nexus_test.log @@ -32,24 +32,34 @@ DEBUG - classpath: ['NXentry'] DEBUG - classes: NXarpes.nxdl.xml:/ENTRY NXentry.nxdl.xml: +NXobject.nxdl.xml: DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY): DEBUG - DEBUG - documentation (NXentry.nxdl.xml:): DEBUG - - (**required**) :ref:`NXentry` describes the measurement. - - The top-level NeXus group which contains all the data and associated - information that comprise a single measurement. - It is mandatory that there is at least one - group of this type in the NeXus file. - + (**required**) :ref:`NXentry` describes the measurement. + + The top-level NeXus group which contains all the data and associated + information that comprise a single measurement. + It is mandatory that there is at least one + group of this type in the NeXus file. +DEBUG - documentation (NXobject.nxdl.xml:): +DEBUG - + This is the base object of NeXus. The groups and fields contained + within this file are allowed to be present in any derived base class. + + If nameType="partial", the placeholders (e.g., FIELDNAME or GROUPNAME) + can be replaced by the name of any object (field or group, + respectively) that exists within the same group. + DEBUG - ===== ATTRS (//entry@NX_class) DEBUG - value: NXentry DEBUG - classpath: ['NXentry'] DEBUG - classes: NXarpes.nxdl.xml:/ENTRY NXentry.nxdl.xml: +NXobject.nxdl.xml: DEBUG - @NX_class [NX_CHAR] DEBUG - DEBUG - ===== FIELD (//entry/collection_time): @@ -60,9 +70,9 @@ NXentry.nxdl.xml:/collection_time DEBUG - <> DEBUG - documentation (NXentry.nxdl.xml:/collection_time): DEBUG - - Time transpired actually collecting data i.e. taking out time when collection was - suspended due to e.g. temperature out of range - + Time transpired actually collecting data i.e. taking out time when collection was + suspended due to e.g. temperature out of range + DEBUG - ===== ATTRS (//entry/collection_time@units) DEBUG - value: s DEBUG - classpath: ['NXentry', 'NX_FLOAT'] @@ -74,158 +84,225 @@ DEBUG - classpath: ['NXentry', 'NXdata'] DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/DATA NXentry.nxdl.xml:/DATA +NXobject.nxdl.xml:/DATA NXdata.nxdl.xml: +NXobject.nxdl.xml: DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/DATA): DEBUG - DEBUG - documentation (NXentry.nxdl.xml:/DATA): DEBUG - - The data group - - .. note:: Before the NIAC2016 meeting [#]_, at least one - :ref:`NXdata` group was required in each :ref:`NXentry` group. - At the NIAC2016 meeting, it was decided to make :ref:`NXdata` - an optional group in :ref:`NXentry` groups for data files that - do not use an application definition. - It is recommended strongly that all NeXus data files provide - a NXdata group. - It is permissable to omit the NXdata group only when - defining the default plot is not practical or possible - from the available data. - - For example, neutron event data may not have anything that - makes a useful plot without extensive processing. - - Certain application definitions override this decision and - require an :ref:`NXdata` group - in the :ref:`NXentry` group. The ``minOccurs=0`` attribute - in the application definition will indicate the - :ref:`NXdata` group - is optional, otherwise, it is required. - - .. [#] NIAC2016: - https://www.nexusformat.org/NIAC2016.html, - https://github.com/nexusformat/NIAC/issues/16 - + The data group + + .. note:: Before the NIAC2016 meeting [#]_, at least one + :ref:`NXdata` group was required in each :ref:`NXentry` group. + At the NIAC2016 meeting, it was decided to make :ref:`NXdata` + an optional group in :ref:`NXentry` groups for data files that + do not use an application definition. + It is recommended strongly that all NeXus data files provide + a NXdata group. + It is permissible to omit the NXdata group only when + defining the default plot is not practical or possible + from the available data. + + For example, neutron event data may not have anything that + makes a useful plot without extensive processing. + + Certain application definitions override this decision and + require an :ref:`NXdata` group + in the :ref:`NXentry` group. The ``minOccurs=0`` attribute + in the application definition will indicate the + :ref:`NXdata` group + is optional, otherwise, it is required. + + .. [#] NIAC2016: + https://www.nexusformat.org/NIAC2016.html, + https://github.com/nexusformat/NIAC/issues/16 + + +DEBUG - documentation (NXobject.nxdl.xml:/DATA): +DEBUG - DEBUG - documentation (NXdata.nxdl.xml:): DEBUG - - :ref:`NXdata` describes the plottable data and related dimension scales. - - .. index:: plotting - - It is strongly recommended that there is at least one :ref:`NXdata` - group in each :ref:`NXentry` group. - Note that the fields named ``AXISNAME`` and ``DATA`` - can be defined with different names. - (Upper case is used to indicate that the actual name is left to the user.) - The ``signal`` and ``axes`` attributes of the - ``data`` group define which items - are plottable data and which are *dimension scales*, respectively. - - :ref:`NXdata` is used to implement one of the basic motivations in NeXus, - to provide a default plot for the data of this :ref:`NXentry`. The actual data - might be stored in another group and (hard) linked to the :ref:`NXdata` group. - - * Each :ref:`NXdata` group will define one field as the default - plottable data. The value of the ``signal`` attribute names this field. - Additional fields may be used to describe the dimension scales and - uncertainities. - The ``auxiliary_signals`` attribute is a list of the other fields - to be plotted with the ``signal`` data. - * The plottable data may be of arbitrary rank up to a maximum - of ``NX_MAXRANK=32`` (for compatibility with backend file formats). - * The plottable data will be named as the value of - the group ``signal`` attribute, such as:: - - data:NXdata - @signal = "counts" - @axes = "mr" - @mr_indices = 0 - counts: float[100] --> the default dependent data - mr: float[100] --> the default independent data - - The field named in the ``signal`` attribute **must** exist, either - directly as a NeXus field or defined through a link. - - * The group ``axes`` attribute will name the - *dimension scale* associated with the plottable data. - - If available, the standard deviations of the data are to be - stored in a data set of the same rank and dimensions, with the name ``errors``. - - * For each data dimension, there should be a one-dimensional array - of the same length. - * These one-dimensional arrays are the *dimension scales* of the - data, *i.e*. the values of the independent variables at which the data - is measured, such as scattering angle or energy transfer. - - .. index:: link - .. index:: axes (attribute) - - The preferred method to associate each data dimension with - its respective dimension scale is to specify the field name - of each dimension scale in the group ``axes`` attribute as a string list. - Here is an example for a 2-D data set *data* plotted - against *time*, and *pressure*. (An additional *temperature* data set - is provided and could be selected as an alternate for the *pressure* axis.):: - - data_2d:NXdata - @signal="data" - @axes=["time", "pressure"] - @pressure_indices=1 - @temperature_indices=1 - @time_indices=0 - data: float[1000,20] - pressure: float[20] - temperature: float[20] - time: float[1000] - - .. rubric:: Old methods to identify the plottable data - - There are two older methods of associating - each data dimension to its respective dimension scale. - Both are now out of date and - should not be used when writing new data files. - However, client software should expect to see data files - written with any of these methods. - - * One method uses the ``axes`` - attribute to specify the names of each *dimension scale*. - - * The oldest method uses the ``axis`` attribute on each - *dimension scale* to identify - with an integer the axis whose value is the number of the dimension. - - .. index: !plot; axis label - plot, axis units - units - dimension scale - - Each axis of the plot may be labeled with information from the - dimension scale for that axis. The optional ``@long_name`` attribute - is provided as the axis label default. If ``@long_name`` is not - defined, then use the name of the dimension scale. A ``@units`` attribute, - if available, may be added to the axis label for further description. - See the section :ref:`Design-Units` for more information. - - .. index: !plot; axis title - - The optional ``title`` field, if available, provides a suggested - title for the plot. If no ``title`` field is found in the :ref:`NXdata` - group, look for a ``title`` field in the parent :ref:`NXentry` group, - with a fallback to displaying the path to the :ref:`NXdata` group. - - NeXus is about how to find and annotate the data to be plotted - but not to describe how the data is to be plotted. - (https://www.nexusformat.org/NIAC2018Minutes.html#nxdata-plottype--attribute) - + The :ref:`NXdata` class is designed to encapsulate all the information required for a set of data to be plotted. + NXdata groups contain plottable data (also referred to as *signals* or *dependent variables*) and their + associated axis coordinates (also referred to as *axes* or *independent variables*). + + The actual names of the :ref:`DATA ` and :ref:`AXISNAME ` fields + can be chosen :ref:`freely `, as indicated by the upper case (this is a common convention in all NeXus classes). + + .. note:: ``NXdata`` provides data and coordinates to be plotted but + does not describe how the data is to be plotted or even the dimensionality of the plot. + https://www.nexusformat.org/NIAC2018Minutes.html#nxdata-plottype--attribute + + .. include:: data/index.rst + :start-line: 1 + + .. admonition:: Example of a simple curve plot + + .. code-block:: + + data:NXdata + @signal = "data" + @axes = ["x"] + data: float[100] + x: float[100] + + More complex cases are supported + + * histogram data: ``x`` has one more value than ``data``. + * alternative axes: instead of a single ``x`` axis you can have several axes, one of which being the default. + * signals with more than one dimension: ``data`` could be 2D with axes ``x`` and ``y`` along each dimension. + * axes with more than one dimension: ``data`` could be 2D with axes ``x`` and ``y`` also being 2D, providing a + unique ``(x, y)`` coordinate for each ``data`` point. + + **Signals:** + + .. index:: plotting + + .. admonition:: Defined by + + * :ref:`DATA ` fields + * the :ref:`signal ` attribute + * the :ref:`auxiliary_signals` attribute + + The :ref:`DATA ` fields contain the signal values to be plotted. The name of the field + to be used as the *default plot signal* is provided by the :ref:`signal ` attribute. + The names of the fields to be used as *secondary plot signals* are provided by the + :ref:`auxiliary_signals` attribute. + + .. admonition:: An example with three signals, one of which being the default + + .. code-block:: + + data:NXdata + @signal = "data1" + @auxiliary_signals = ["data2", "data3"] + data1: float[10,20,30] # the default signal + data2: float[10,20,30] + data3: float[10,20,30] + + **Axes:** + + .. index:: axes (attribute) + .. index:: coordinates + + .. admonition:: Defined by + + * :ref:`AXISNAME ` fields + * the :ref:`axes ` attribute + * :ref:`AXISNAME_indices ` attributes + + The fields and attributes are defined as follows + + 1. The :ref:`AXISNAME ` fields contain the axis coordinates associated with the signal values. + + 2. The :ref:`axes ` attribute provides the names of the :ref:`AXISNAME ` + fields to be used as the `default axis` for each dimension of the :ref:`DATA ` fields. + + 3. The :ref:`AXISNAME_indices ` attributes describe the :ref:`DATA ` + dimensions spanned by the corresponding :ref:`AXISNAME ` fields. + + The fields and attributes have the following constraints + + 1. The length of the :ref:`axes ` attribute must be equal to the rank of the :ref:`DATA ` + fields. When a particular dimension has no default axis, the string “.” is used in that position. + + 2. The number of values in :ref:`AXISNAME_indices ` must be equal to the rank of the corresponding + :ref:`AXISNAME ` field. + + 3. When :ref:`AXISNAME_indices ` is missing for a given + :ref:`AXISNAME ` field, the positions of the :ref:`AXISNAME ` + field name in the :ref:`axes ` attribute are used. + + 4. When :ref:`AXISNAME_indices ` is the same as the indices of "AXISNAME" in the + :ref:`axes ` attribute, there is no need to provide + :ref:`AXISNAME_indices `. + + 5. The indices of "AXISNAME" in the :ref:`axes ` attribute must be a subset of + :ref:`AXISNAME_indices `. + + 6. The shape of an :ref:`AXISNAME ` field must correspond to the shape of the + :ref:`DATA ` dimensions it spans. This means that for each dimension ``i`` in ``[0, AXISNAME.ndim)`` + spanned by axis field :ref:`AXISNAME `, the number of axis values ``AXISNAME.shape[i]`` + along dimension ``i`` must be equal to the number of data points ``DATA.shape[AXISNAME_indices[i]]`` along dimension ``i`` + or one more than the number of data points ``DATA.shape[AXISNAME_indices[i]]+1`` in case the + :ref:`AXISNAME ` field contains histogram bin edges along dimension ``i``. + + Highlight consequences of these constraints + + 1. An :ref:`AXISNAME ` field can have more than one dimension and can therefore span + more than one :ref:`DATA ` dimension. Conversely, one :ref:`DATA ` dimension + can be spanned by more than one :ref:`AXISNAME ` field. The default axis name (if any) + of each dimension can be found in the :ref:`axes ` attribute. + + 2. A list of all available axes is not provided directly. All strings in the :ref:`axes ` attribute + (excluding the “.” string) are axis field names. In addition the prefix of an attribute ending with the string "_indices" is also + an axis field name. + + .. admonition:: The following example covers all axes features supported (see :ref:`sphx_glr_classes_base_classes_data_plot_fscan2d.py`) + + .. code-block:: + + data:NXdata + @signal = "data" + @axes = ["x_set", "y_set", "."] # default axes for all three dimensions + @x_encoder_indices = [0, 1] + @y_encoder_indices = 1 # or [1] + data: float[10,7,1024] + x_encoder: float[11,7] # coordinates along the first and second dimensions + y_encoder: float[7] # coordinates along the second dimension + x_set: float[10] # default coordinates along the first dimension + y_set: float[7] # default coordinates along the second dimension + + **Uncertainties:** + + .. admonition:: Defined by + + * :ref:`FIELDNAME_errors ` fields + + Standard deviations on data values as well as coordinates can be provided by + :ref:`FIELDNAME_errors ` fields where ``FIELDNAME`` is the name of a + :ref:`DATA ` field or an :ref:`AXISNAME ` field. + + .. admonition:: An example of uncertainties on the signal, auxiliary signals and axis coordinates + + .. code-block:: + + data:NXdata + @signal = "data1" + @auxiliary_signals = ["data2", "data3"] + @axes = ["x", ".", "z"] + data1: float[10,20,30] + data2: float[10,20,30] + data3: float[10,20,30] + x: float[10] + z: float[30] + data1_errors: float[10,20,30] + data2_errors: float[10,20,30] + data3_errors: float[10,20,30] + x_errors: float[10] + z_errors: float[30] + + +DEBUG - documentation (NXobject.nxdl.xml:): +DEBUG - + This is the base object of NeXus. The groups and fields contained + within this file are allowed to be present in any derived base class. + + If nameType="partial", the placeholders (e.g., FIELDNAME or GROUPNAME) + can be replaced by the name of any object (field or group, + respectively) that exists within the same group. + DEBUG - ===== ATTRS (//entry/data@NX_class) DEBUG - value: NXdata DEBUG - classpath: ['NXentry', 'NXdata'] DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/DATA NXentry.nxdl.xml:/DATA +NXobject.nxdl.xml:/DATA NXdata.nxdl.xml: +NXobject.nxdl.xml: DEBUG - @NX_class [NX_CHAR] DEBUG - DEBUG - ===== ATTRS (//entry/data@axes) @@ -234,96 +311,89 @@ DEBUG - classpath: ['NXentry', 'NXdata'] DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/DATA NXentry.nxdl.xml:/DATA +NXobject.nxdl.xml:/DATA NXdata.nxdl.xml: +NXobject.nxdl.xml: DEBUG - NXdata.nxdl.xml:@axes - [NX_CHAR] DEBUG - <> DEBUG - documentation (NXdata.nxdl.xml:/axes): DEBUG - - .. index:: plotting - - Array of strings holding the :ref:`names ` of - the independent data fields used in the default plot for all of - the dimensions of the :ref:`signal ` - as well as any :ref:`auxiliary signals `. - - One name is provided for every dimension in the *signal* or *auxiliary signal* fields. - - The *axes* values are the names of fields or links that *must* exist and be direct - children of this NXdata group. - - An axis slice is specified using a field named ``AXISNAME_indices`` - as described below (where the text shown here as ``AXISNAME`` is to be - replaced by the actual field name). - - When no default axis is available for a particular dimension - of the plottable data, use a "." in that position. - Such as:: - - @axes=["time", ".", "."] - - Since there are three items in the list, the *signal* field - must be a three-dimensional array (rank=3). The first dimension - is described by the values of a one-dimensional array named ``time`` - while the other two dimensions have no fields to be used as dimension scales. - - See examples provided on the NeXus wiki: - https://www.nexusformat.org/2014_axes_and_uncertainties.html - - If there are no axes at all (such as with a stack of images), - the axes attribute can be omitted. - + .. index:: plotting + + The ``axes`` attribute is a list of strings which are the names of the :ref:`AXISNAME ` fields + to be used as the default axis along every :ref:`DATA ` dimension. As a result the length must + be equal to the rank of the :ref:`DATA ` fields. The string "." can be used for + dimensions without a default axis. + + .. note:: When ``axes`` contains multiple strings, it must be saved as an actual array + of strings and not a single comma separated string. + DEBUG - ===== ATTRS (//entry/data@signal) DEBUG - value: data DEBUG - classpath: ['NXentry', 'NXdata'] DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/DATA NXentry.nxdl.xml:/DATA +NXobject.nxdl.xml:/DATA NXdata.nxdl.xml: +NXobject.nxdl.xml: DEBUG - NXdata.nxdl.xml:@signal - [NX_CHAR] DEBUG - <> DEBUG - documentation (NXdata.nxdl.xml:/signal): DEBUG - - .. index:: find the default plottable data - .. index:: plotting - .. index:: signal attribute value - - Declares which NeXus field is the default. - The value is the :ref:`name ` of the data field to be plotted. - This field or link *must* exist and be a direct child of this NXdata group. - - It is recommended (as of NIAC2014) to use this attribute - rather than adding a signal attribute to the field. - See https://www.nexusformat.org/2014_How_to_find_default_data.html - for a summary of the discussion. - + .. index:: find the default plottable data + .. index:: plotting + .. index:: signal attribute value + + The value is the :ref:`name ` of the signal that contains + the default plottable data. This field or link *must* exist and be a direct child + of this NXdata group. + + It is recommended (as of NIAC2014) to use this attribute + rather than adding a signal attribute to the field. + See https://www.nexusformat.org/2014_How_to_find_default_data.html + for a summary of the discussion. + DEBUG - ===== FIELD (//entry/data/angles): DEBUG - value: [-1.96735314 -1.91500657 -1.86266001 -1.81031344 -1.75796688 -1.70562031 ... -DEBUG - classpath: ['NXentry', 'NXdata', 'NX_NUMBER'] +DEBUG - classpath: ['NXentry', 'NXdata', 'NX_CHAR_OR_NUMBER'] DEBUG - classes: NXdata.nxdl.xml:/AXISNAME DEBUG - <> DEBUG - Dataset referenced as NXdata AXIS #0 DEBUG - documentation (NXdata.nxdl.xml:/AXISNAME): DEBUG - - Dimension scale defining an axis of the data. - Client is responsible for defining the dimensions of the data. - The name of this field may be changed to fit the circumstances. - Standard NeXus client tools will use the attributes to determine - how to use this field. - + Coordinate values along one or more :ref:`DATA ` dimensions. + + The shape of an ``AXISNAME`` field must correspond to the shape of the :ref:`DATA ` + dimensions it spans. This means that for each ``i`` in ``[0, AXISNAME.ndim)`` the number of data points + ``DATA.shape[AXISNAME_indices[i]]`` must be equal to the number of coordinates ``AXISNAME.shape[i]`` or the + number of bin edges ``AXISNAME.shape[i]+1`` in case of histogram data. + + As the upper case ``AXISNAME`` indicates, the names of the ``AXISNAME`` fields can be chosen :ref:`freely `. + + Most ``AXISNAME`` fields will be sequences of numbers but if an axis is better represented using names, such as channel names, + an array of NX_CHAR can be provided. + DEBUG - ===== ATTRS (//entry/data/angles@target) DEBUG - value: /entry/instrument/analyser/angles -DEBUG - classpath: ['NXentry', 'NXdata', 'NX_NUMBER'] +DEBUG - classpath: ['NXentry', 'NXdata', 'NX_CHAR_OR_NUMBER'] DEBUG - classes: NXdata.nxdl.xml:/AXISNAME DEBUG - @target - IS NOT IN SCHEMA DEBUG - DEBUG - ===== ATTRS (//entry/data/angles@units) DEBUG - value: 1/Å -DEBUG - classpath: ['NXentry', 'NXdata', 'NX_NUMBER'] +DEBUG - classpath: ['NXentry', 'NXdata', 'NX_CHAR_OR_NUMBER'] DEBUG - classes: NXdata.nxdl.xml:/AXISNAME -DEBUG - NXdata.nxdl.xml:/AXISNAME@units - REQUIRED, but undefined unit category +DEBUG - NXdata.nxdl.xml:/AXISNAME@units - [NX_CHAR] +DEBUG - Dataset referenced as NXdata AXIS #0 +DEBUG - documentation (NXdata.nxdl.xml:/AXISNAME/units): +DEBUG - + Unit in which the coordinate values are expressed. + See the section :ref:`Design-Units` for more information. + DEBUG - ===== FIELD (//entry/data/data): DEBUG - value: [[0. 0. 0. ... 0. 0. 0.] ... DEBUG - classpath: ['NXentry', 'NXdata', 'NX_NUMBER'] @@ -333,15 +403,15 @@ DEBUG - <> DEBUG - Dataset referenced as NXdata SIGNAL DEBUG - documentation (NXdata.nxdl.xml:/DATA): DEBUG - - .. index:: plotting - - This field contains the data values to be used as the - NeXus *plottable data*. - Client is responsible for defining the dimensions of the data. - The name of this field may be changed to fit the circumstances. - Standard NeXus client tools will use the attributes to determine - how to use this field. - + .. index:: plotting + + Data values to be used as the NeXus *plottable data*. As the upper case ``DATA`` + indicates, the names of the ``DATA`` fields can be chosen :ref:`freely `. The :ref:`signal attribute ` + and :ref:`auxiliary_signals attribute` can be used to find all datasets in the ``NXdata`` + that contain data values. + + The maximum rank is ``32`` for compatibility with backend file formats. + DEBUG - ===== ATTRS (//entry/data/data@target) DEBUG - value: /entry/instrument/analyser/data DEBUG - classpath: ['NXentry', 'NXdata', 'NX_NUMBER'] @@ -357,60 +427,84 @@ NXdata.nxdl.xml:/DATA DEBUG - NXdata.nxdl.xml:/DATA@units - REQUIRED, but undefined unit category DEBUG - ===== FIELD (//entry/data/delays): DEBUG - value: [-1.1 -1.08041237 -1.06082474 -1.04123711 -1.02164948 -1.00206186 ... -DEBUG - classpath: ['NXentry', 'NXdata', 'NX_NUMBER'] +DEBUG - classpath: ['NXentry', 'NXdata', 'NX_CHAR_OR_NUMBER'] DEBUG - classes: NXdata.nxdl.xml:/AXISNAME DEBUG - <> DEBUG - Dataset referenced as NXdata AXIS #2 DEBUG - documentation (NXdata.nxdl.xml:/AXISNAME): DEBUG - - Dimension scale defining an axis of the data. - Client is responsible for defining the dimensions of the data. - The name of this field may be changed to fit the circumstances. - Standard NeXus client tools will use the attributes to determine - how to use this field. - + Coordinate values along one or more :ref:`DATA ` dimensions. + + The shape of an ``AXISNAME`` field must correspond to the shape of the :ref:`DATA ` + dimensions it spans. This means that for each ``i`` in ``[0, AXISNAME.ndim)`` the number of data points + ``DATA.shape[AXISNAME_indices[i]]`` must be equal to the number of coordinates ``AXISNAME.shape[i]`` or the + number of bin edges ``AXISNAME.shape[i]+1`` in case of histogram data. + + As the upper case ``AXISNAME`` indicates, the names of the ``AXISNAME`` fields can be chosen :ref:`freely `. + + Most ``AXISNAME`` fields will be sequences of numbers but if an axis is better represented using names, such as channel names, + an array of NX_CHAR can be provided. + DEBUG - ===== ATTRS (//entry/data/delays@target) DEBUG - value: /entry/instrument/analyser/delays -DEBUG - classpath: ['NXentry', 'NXdata', 'NX_NUMBER'] +DEBUG - classpath: ['NXentry', 'NXdata', 'NX_CHAR_OR_NUMBER'] DEBUG - classes: NXdata.nxdl.xml:/AXISNAME DEBUG - @target - IS NOT IN SCHEMA DEBUG - DEBUG - ===== ATTRS (//entry/data/delays@units) DEBUG - value: fs -DEBUG - classpath: ['NXentry', 'NXdata', 'NX_NUMBER'] +DEBUG - classpath: ['NXentry', 'NXdata', 'NX_CHAR_OR_NUMBER'] DEBUG - classes: NXdata.nxdl.xml:/AXISNAME -DEBUG - NXdata.nxdl.xml:/AXISNAME@units - REQUIRED, but undefined unit category +DEBUG - NXdata.nxdl.xml:/AXISNAME@units - [NX_CHAR] +DEBUG - Dataset referenced as NXdata AXIS #2 +DEBUG - documentation (NXdata.nxdl.xml:/AXISNAME/units): +DEBUG - + Unit in which the coordinate values are expressed. + See the section :ref:`Design-Units` for more information. + DEBUG - ===== FIELD (//entry/data/energies): DEBUG - value: [ 2.5 2.46917808 2.43835616 2.40753425 2.37671233 2.34589041 ... -DEBUG - classpath: ['NXentry', 'NXdata', 'NX_NUMBER'] +DEBUG - classpath: ['NXentry', 'NXdata', 'NX_CHAR_OR_NUMBER'] DEBUG - classes: NXdata.nxdl.xml:/AXISNAME DEBUG - <> DEBUG - Dataset referenced as NXdata AXIS #1 DEBUG - documentation (NXdata.nxdl.xml:/AXISNAME): DEBUG - - Dimension scale defining an axis of the data. - Client is responsible for defining the dimensions of the data. - The name of this field may be changed to fit the circumstances. - Standard NeXus client tools will use the attributes to determine - how to use this field. - + Coordinate values along one or more :ref:`DATA ` dimensions. + + The shape of an ``AXISNAME`` field must correspond to the shape of the :ref:`DATA ` + dimensions it spans. This means that for each ``i`` in ``[0, AXISNAME.ndim)`` the number of data points + ``DATA.shape[AXISNAME_indices[i]]`` must be equal to the number of coordinates ``AXISNAME.shape[i]`` or the + number of bin edges ``AXISNAME.shape[i]+1`` in case of histogram data. + + As the upper case ``AXISNAME`` indicates, the names of the ``AXISNAME`` fields can be chosen :ref:`freely `. + + Most ``AXISNAME`` fields will be sequences of numbers but if an axis is better represented using names, such as channel names, + an array of NX_CHAR can be provided. + DEBUG - ===== ATTRS (//entry/data/energies@target) DEBUG - value: /entry/instrument/analyser/energies -DEBUG - classpath: ['NXentry', 'NXdata', 'NX_NUMBER'] +DEBUG - classpath: ['NXentry', 'NXdata', 'NX_CHAR_OR_NUMBER'] DEBUG - classes: NXdata.nxdl.xml:/AXISNAME DEBUG - @target - IS NOT IN SCHEMA DEBUG - DEBUG - ===== ATTRS (//entry/data/energies@units) DEBUG - value: eV -DEBUG - classpath: ['NXentry', 'NXdata', 'NX_NUMBER'] +DEBUG - classpath: ['NXentry', 'NXdata', 'NX_CHAR_OR_NUMBER'] DEBUG - classes: NXdata.nxdl.xml:/AXISNAME -DEBUG - NXdata.nxdl.xml:/AXISNAME@units - REQUIRED, but undefined unit category +DEBUG - NXdata.nxdl.xml:/AXISNAME@units - [NX_CHAR] +DEBUG - Dataset referenced as NXdata AXIS #1 +DEBUG - documentation (NXdata.nxdl.xml:/AXISNAME/units): +DEBUG - + Unit in which the coordinate values are expressed. + See the section :ref:`Design-Units` for more information. + DEBUG - ===== FIELD (//entry/definition): DEBUG - value: NXarpes DEBUG - classpath: ['NXentry', 'NX_CHAR'] @@ -421,26 +515,24 @@ DEBUG - <> DEBUG - enumeration (NXarpes.nxdl.xml:/ENTRY/definition): DEBUG - -> NXarpes DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/definition): -DEBUG - - Official NeXus NXDL schema to which this file conforms. - +DEBUG - Official NeXus NXDL schema to which this file conforms. DEBUG - documentation (NXentry.nxdl.xml:/definition): DEBUG - - (alternate use: see same field in :ref:`NXsubentry` for preferred) - - Official NeXus NXDL schema to which this entry conforms which must be - the name of the NXDL file (case sensitive without the file extension) - that the NXDL schema is defined in. - - For example the ``definition`` field for a file that conformed to the - *NXarpes.nxdl.xml* definition must contain the string **NXarpes**. - - This field is provided so that :ref:`NXentry` can be the overlay position - in a NeXus data file for an application definition and its - set of groups, fields, and attributes. - - *It is advised* to use :ref:`NXsubentry`, instead, as the overlay position. - + (alternate use: see same field in :ref:`NXsubentry` for preferred) + + Official NeXus NXDL schema to which this entry conforms which must be + the name of the NXDL file (case sensitive without the file extension) + that the NXDL schema is defined in. + + For example the ``definition`` field for a file that conformed to the + *NXarpes.nxdl.xml* definition must contain the string **NXarpes**. + + This field is provided so that :ref:`NXentry` can be the overlay position + in a NeXus data file for an application definition and its + set of groups, fields, and attributes. + + *It is advised* to use :ref:`NXsubentry`, instead, as the overlay position. + DEBUG - ===== FIELD (//entry/duration): DEBUG - value: 7200 DEBUG - classpath: ['NXentry', 'NX_INT'] @@ -448,9 +540,7 @@ DEBUG - classes: NXentry.nxdl.xml:/duration DEBUG - <> DEBUG - documentation (NXentry.nxdl.xml:/duration): -DEBUG - - Duration of measurement - +DEBUG - Duration of measurement DEBUG - ===== ATTRS (//entry/duration@units) DEBUG - value: s DEBUG - classpath: ['NXentry', 'NX_INT'] @@ -464,25 +554,36 @@ DEBUG - classes: NXentry.nxdl.xml:/end_time DEBUG - <> DEBUG - documentation (NXentry.nxdl.xml:/end_time): -DEBUG - - Ending time of measurement - +DEBUG - Ending time of measurement DEBUG - ===== FIELD (//entry/entry_identifier): DEBUG - value: Run 22118 -DEBUG - classpath: ['NXentry'] -DEBUG - NOT IN SCHEMA -DEBUG - +DEBUG - classpath: ['NXentry', 'NX_CHAR'] +DEBUG - classes: +NXentry.nxdl.xml:/entry_identifier +DEBUG - <> +DEBUG - DEPRECATED - Use the field :ref:`identifier_entry ` instead. +DEBUG - documentation (NXentry.nxdl.xml:/entry_identifier): +DEBUG - unique identifier for the measurement, defined by the facility. DEBUG - ===== FIELD (//entry/experiment_identifier): DEBUG - value: F-20170538 -DEBUG - classpath: ['NXentry'] -DEBUG - NOT IN SCHEMA +DEBUG - classpath: ['NXentry', 'NX_CHAR'] +DEBUG - classes: +NXentry.nxdl.xml:/experiment_identifier +DEBUG - <> +DEBUG - DEPRECATED - Use the field :ref:`identifier_experiment ` instead. +DEBUG - documentation (NXentry.nxdl.xml:/experiment_identifier): DEBUG - + Unique identifier for the experiment, + defined by the facility, + possibly linked to the proposals + DEBUG - ===== GROUP (//entry/instrument [NXarpes::/NXentry/NXinstrument]): DEBUG - classpath: ['NXentry', 'NXinstrument'] DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT NXentry.nxdl.xml:/INSTRUMENT NXinstrument.nxdl.xml: +NXobject.nxdl.xml: DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT): DEBUG - @@ -490,15 +591,24 @@ DEBUG - documentation (NXentry.nxdl.xml:/INSTRUMENT): DEBUG - DEBUG - documentation (NXinstrument.nxdl.xml:): DEBUG - - Collection of the components of the instrument or beamline. - - Template of instrument descriptions comprising various beamline components. - Each component will also be a NeXus group defined by its distance from the - sample. Negative distances represent beamline components that are before the - sample while positive distances represent components that are after the sample. - This device allows the unique identification of beamline components in a way - that is valid for both reactor and pulsed instrumentation. - + Collection of the components of the instrument or beamline. + + Template of instrument descriptions comprising various beamline components. + Each component will also be a NeXus group defined by its distance from the + sample. Negative distances represent beamline components that are before the + sample while positive distances represent components that are after the sample. + This device allows the unique identification of beamline components in a way + that is valid for both reactor and pulsed instrumentation. + +DEBUG - documentation (NXobject.nxdl.xml:): +DEBUG - + This is the base object of NeXus. The groups and fields contained + within this file are allowed to be present in any derived base class. + + If nameType="partial", the placeholders (e.g., FIELDNAME or GROUPNAME) + can be replaced by the name of any object (field or group, + respectively) that exists within the same group. + DEBUG - ===== ATTRS (//entry/instrument@NX_class) DEBUG - value: NXinstrument DEBUG - classpath: ['NXentry', 'NXinstrument'] @@ -506,6 +616,7 @@ DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT NXentry.nxdl.xml:/INSTRUMENT NXinstrument.nxdl.xml: +NXobject.nxdl.xml: DEBUG - @NX_class [NX_CHAR] DEBUG - DEBUG - ===== GROUP (//entry/instrument/analyser [NXarpes::/NXentry/NXinstrument/NXdetector]): @@ -514,6 +625,8 @@ DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser NXinstrument.nxdl.xml:/DETECTOR NXdetector.nxdl.xml: +NXcomponent.nxdl.xml: +NXobject.nxdl.xml: DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser): DEBUG - @@ -521,8 +634,21 @@ DEBUG - documentation (NXinstrument.nxdl.xml:/DETECTOR): DEBUG - DEBUG - documentation (NXdetector.nxdl.xml:): DEBUG - - A detector, detector bank, or multidetector. + A detector, detector bank, or multidetector. + +DEBUG - documentation (NXcomponent.nxdl.xml:): +DEBUG - + Base class for components of an instrument - real ones or simulated ones. +DEBUG - documentation (NXobject.nxdl.xml:): +DEBUG - + This is the base object of NeXus. The groups and fields contained + within this file are allowed to be present in any derived base class. + + If nameType="partial", the placeholders (e.g., FIELDNAME or GROUPNAME) + can be replaced by the name of any object (field or group, + respectively) that exists within the same group. + DEBUG - ===== ATTRS (//entry/instrument/analyser@NX_class) DEBUG - value: NXdetector DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector'] @@ -530,6 +656,8 @@ DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser NXinstrument.nxdl.xml:/DETECTOR NXdetector.nxdl.xml: +NXcomponent.nxdl.xml: +NXobject.nxdl.xml: DEBUG - @NX_class [NX_CHAR] DEBUG - DEBUG - ===== FIELD (//entry/instrument/analyser/acquisition_mode): @@ -553,19 +681,12 @@ DEBUG - -> pulse counting DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/acquisition_mode): DEBUG - DEBUG - documentation (NXdetector.nxdl.xml:/acquisition_mode): -DEBUG - - The acquisition mode of the detector. - +DEBUG - The acquisition mode of the detector. DEBUG - ===== FIELD (//entry/instrument/analyser/amplifier_type): DEBUG - value: MCP -DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector', 'NX_CHAR'] -DEBUG - classes: -NXdetector.nxdl.xml:/amplifier_type -DEBUG - <> -DEBUG - documentation (NXdetector.nxdl.xml:/amplifier_type): +DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector'] +DEBUG - NOT IN SCHEMA DEBUG - - Type of electron amplifier, MCP, channeltron, etc. - DEBUG - ===== FIELD (//entry/instrument/analyser/angles): DEBUG - value: [-1.96735314 -1.91500657 -1.86266001 -1.81031344 -1.75796688 -1.70562031 ... DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector', 'NX_NUMBER'] @@ -574,10 +695,10 @@ NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/angles DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/angles): DEBUG - - Angular axis of the analyser data - which dimension the axis applies to is defined - using the normal NXdata methods. - + Angular axis of the analyser data + which dimension the axis applies to is defined + using the normal NXdata methods. + DEBUG - ===== ATTRS (//entry/instrument/analyser/angles@target) DEBUG - value: /entry/instrument/analyser/angles DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector', 'NX_NUMBER'] @@ -607,29 +728,29 @@ DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/data): DEBUG - DEBUG - documentation (NXdetector.nxdl.xml:/data): DEBUG - - Data values from the detector. The rank and dimension ordering should follow a principle of - slowest to fastest measurement axes and may be explicitly specified in application definitions. - - Mechanical scanning of objects (e.g. sample position/angle, incident beam energy, etc) tends to be - the slowest part of an experiment and so any such scan axes should be allocated to the first dimensions - of the array. Note that in some cases it may be useful to represent a 2D set of scan points as a single - scan-axis in the data array, especially if the scan pattern doesn't fit a rectangular array nicely. - Repetition of an experiment in a time series tends to be used similar to a slow scan axis - and so will often be in the first dimension of the data array. - - The next fastest axes are typically the readout of the detector. A point detector will not add any dimensions - (as it is just a single value per scan point) to the data array, a strip detector will add one dimension, an - imaging detector will add two dimensions (e.g. X, Y axes) and detectors outputting higher dimensional data - will add the corresponding number of dimensions. Note that the detector dimensions don't necessarily have to - be written in order of the actual readout speeds - the slowest to fastest rule principle is only a guide. - - Finally, detectors that operate in a time-of-flight mode, such as a neutron spectrometer or a silicon drift - detector (used for X-ray fluorescence) tend to have their dimension(s) added to the last dimensions in the data array. - - The type of each dimension should should follow the order of scan points, detector pixels, - then time-of-flight (i.e. spectroscopy, spectrometry). The rank and dimension sizes (see symbol list) - shown here are merely illustrative of coordination between related datasets. - + Data values from the detector. The rank and dimension ordering should follow a principle of + slowest to fastest measurement axes and may be explicitly specified in application definitions. + + Mechanical scanning of objects (e.g. sample position/angle, incident beam energy, etc) tends to be + the slowest part of an experiment and so any such scan axes should be allocated to the first dimensions + of the array. Note that in some cases it may be useful to represent a 2D set of scan points as a single + scan-axis in the data array, especially if the scan pattern doesn't fit a rectangular array nicely. + Repetition of an experiment in a time series tends to be used similar to a slow scan axis + and so will often be in the first dimension of the data array. + + The next fastest axes are typically the readout of the detector. A point detector will not add any dimensions + (as it is just a single value per scan point) to the data array, a strip detector will add one dimension, an + imaging detector will add two dimensions (e.g. X, Y axes) and detectors outputting higher dimensional data + will add the corresponding number of dimensions. Note that the detector dimensions don't necessarily have to + be written in order of the actual readout speeds - the slowest to fastest rule principle is only a guide. + + Finally, detectors that operate in a time-of-flight mode, such as a neutron spectrometer or a silicon drift + detector (used for X-ray fluorescence) tend to have their dimension(s) added to the last dimensions in the data array. + + The type of each dimension should should follow the order of scan points, detector pixels, + then time-of-flight (i.e. spectroscopy, spectrometry). The rank and dimension sizes (see symbol list) + shown here are merely illustrative of coordination between related datasets. + DEBUG - ===== ATTRS (//entry/instrument/analyser/data@target) DEBUG - value: /entry/instrument/analyser/data DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector', 'NX_NUMBER'] @@ -663,14 +784,9 @@ DEBUG - NOT IN SCHEMA DEBUG - DEBUG - ===== FIELD (//entry/instrument/analyser/detector_type): DEBUG - value: DLD -DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector', 'NX_CHAR'] -DEBUG - classes: -NXdetector.nxdl.xml:/detector_type -DEBUG - <> -DEBUG - documentation (NXdetector.nxdl.xml:/detector_type): +DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector'] +DEBUG - NOT IN SCHEMA DEBUG - - Description of the detector type, DLD, Phosphor+CCD, CMOS. - DEBUG - ===== FIELD (//entry/instrument/analyser/dispersion_scheme): DEBUG - value: Time of flight DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector'] @@ -684,10 +800,10 @@ NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/energies DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/energies): DEBUG - - Energy axis of the analyser data - which dimension the axis applies to is defined - using the normal NXdata methods. - + Energy axis of the analyser data + which dimension the axis applies to is defined + using the normal NXdata methods. + DEBUG - ===== ATTRS (//entry/instrument/analyser/energies@target) DEBUG - value: /entry/instrument/analyser/energies DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector', 'NX_NUMBER'] @@ -708,9 +824,7 @@ DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/entrance_slit_setting DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/entrance_slit_setting): -DEBUG - - dial setting of the entrance slit - +DEBUG - dial setting of the entrance slit DEBUG - ===== FIELD (//entry/instrument/analyser/entrance_slit_shape): DEBUG - value: straight DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector', 'NX_CHAR'] @@ -729,9 +843,7 @@ DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/entrance_slit_size DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/entrance_slit_size): -DEBUG - - size of the entrance slit - +DEBUG - size of the entrance slit DEBUG - ===== ATTRS (//entry/instrument/analyser/entrance_slit_size@units) DEBUG - value: um DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector', 'NX_NUMBER'] @@ -775,9 +887,7 @@ DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/lens_mode DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/lens_mode): -DEBUG - - setting for the electron analyser lens - +DEBUG - setting for the electron analyser lens DEBUG - ===== FIELD (//entry/instrument/analyser/magnification): DEBUG - value: -1.5 DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector'] @@ -790,9 +900,7 @@ DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/pass_energy DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/pass_energy): -DEBUG - - energy of the electrons on the mean path of the analyser - +DEBUG - energy of the electrons on the mean path of the analyser DEBUG - ===== ATTRS (//entry/instrument/analyser/pass_energy@units) DEBUG - value: eV DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector', 'NX_NUMBER'] @@ -811,9 +919,7 @@ DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/region_origin DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/region_origin): -DEBUG - - origin of rectangular region selected for readout - +DEBUG - origin of rectangular region selected for readout DEBUG - ===== FIELD (//entry/instrument/analyser/region_size): DEBUG - value: [ 80 146] DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector', 'NX_INT'] @@ -821,19 +927,12 @@ DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/region_size DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/region_size): -DEBUG - - size of rectangular region selected for readout - +DEBUG - size of rectangular region selected for readout DEBUG - ===== FIELD (//entry/instrument/analyser/sensor_count): DEBUG - value: 4 -DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector', 'NX_INT'] -DEBUG - classes: -NXdetector.nxdl.xml:/sensor_count -DEBUG - <> -DEBUG - documentation (NXdetector.nxdl.xml:/sensor_count): +DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector'] +DEBUG - NOT IN SCHEMA DEBUG - - Number of imaging sensor chips on the detector. - DEBUG - ===== FIELD (//entry/instrument/analyser/sensor_size): DEBUG - value: [ 80 146] DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector', 'NX_INT'] @@ -841,9 +940,7 @@ DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/sensor_size DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/sensor_size): -DEBUG - - number of raw active elements in each dimension - +DEBUG - number of raw active elements in each dimension DEBUG - ===== FIELD (//entry/instrument/analyser/time_per_channel): DEBUG - value: 7200 DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector', 'NX_NUMBER'] @@ -851,9 +948,7 @@ DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/time_per_channel DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/analyser/time_per_channel): -DEBUG - - todo: define more clearly - +DEBUG - todo: define more clearly DEBUG - ===== ATTRS (//entry/instrument/analyser/time_per_channel@units) DEBUG - value: s DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXdetector', 'NX_NUMBER'] @@ -875,33 +970,44 @@ DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXbeam'] DEBUG - classes: NXinstrument.nxdl.xml:/BEAM NXbeam.nxdl.xml: +NXobject.nxdl.xml: DEBUG - <> DEBUG - documentation (NXinstrument.nxdl.xml:/BEAM): DEBUG - DEBUG - documentation (NXbeam.nxdl.xml:): DEBUG - - Properties of the neutron or X-ray beam at a given location. - - This group is intended to be referenced - by beamline component groups within the :ref:`NXinstrument` group or by the :ref:`NXsample` group. This group is - especially valuable in storing the results of instrument simulations in which it is useful - to specify the beam profile, time distribution etc. at each beamline component. Otherwise, - its most likely use is in the :ref:`NXsample` group in which it defines the results of the neutron - scattering by the sample, e.g., energy transfer, polarizations. Finally, There are cases where the beam is - considered as a beamline component and this group may be defined as a subgroup directly inside - :ref:`NXinstrument`, in which case it is recommended that the position of the beam is specified by an - :ref:`NXtransformations` group, unless the beam is at the origin (which is the sample). - - Note that incident_wavelength and related fields can be a scalar values or arrays, depending on the use case. - To support these use cases, the explicit dimensionality of these fields is not specified, but it can be inferred - by the presense of and shape of accompanying fields, such as incident_wavelength_weights for a polychromatic beam. - + Properties of the neutron or X-ray beam at a given location. + + This group is intended to be referenced + by beamline component groups within the :ref:`NXinstrument` group or by the :ref:`NXsample` group. This group is + especially valuable in storing the results of instrument simulations in which it is useful + to specify the beam profile, time distribution etc. at each beamline component. Otherwise, + its most likely use is in the :ref:`NXsample` group in which it defines the results of the neutron + scattering by the sample, e.g., energy transfer, polarizations. Finally, There are cases where the beam is + considered as a beamline component and this group may be defined as a subgroup directly inside + :ref:`NXinstrument`, in which case it is recommended that the position of the beam is specified by an + :ref:`NXtransformations` group, unless the beam is at the origin (which is the sample). + + Note that ``incident_wavelength``, ``incident_energy``, and related fields can be a scalar values or arrays, depending on the use case. + To support these use cases, the explicit dimensionality of these fields is not specified, but it can be inferred + by the presence of and shape of accompanying fields, such as incident_wavelength_weights for a polychromatic beam. + +DEBUG - documentation (NXobject.nxdl.xml:): +DEBUG - + This is the base object of NeXus. The groups and fields contained + within this file are allowed to be present in any derived base class. + + If nameType="partial", the placeholders (e.g., FIELDNAME or GROUPNAME) + can be replaced by the name of any object (field or group, + respectively) that exists within the same group. + DEBUG - ===== ATTRS (//entry/instrument/beam_probe_0@NX_class) DEBUG - value: NXbeam DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXbeam'] DEBUG - classes: NXinstrument.nxdl.xml:/BEAM NXbeam.nxdl.xml: +NXobject.nxdl.xml: DEBUG - @NX_class [NX_CHAR] DEBUG - DEBUG - ===== FIELD (//entry/instrument/beam_probe_0/distance): @@ -911,9 +1017,7 @@ DEBUG - classes: NXbeam.nxdl.xml:/distance DEBUG - <> DEBUG - documentation (NXbeam.nxdl.xml:/distance): -DEBUG - - Distance from sample. Note, it is recommended to use NXtransformations instead. - +DEBUG - Distance from sample. Note, it is recommended to use NXtransformations instead. DEBUG - ===== ATTRS (//entry/instrument/beam_probe_0/distance@units) DEBUG - value: cm DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXbeam', 'NX_FLOAT'] @@ -953,7 +1057,7 @@ NXbeam.nxdl.xml:/pulse_duration DEBUG - <> DEBUG - documentation (NXbeam.nxdl.xml:/pulse_duration): DEBUG - - FWHM duration of the pulses at the diagnostic point + FWHM duration of the pulses at the given location. DEBUG - ===== ATTRS (//entry/instrument/beam_probe_0/pulse_duration@units) DEBUG - value: fs @@ -986,33 +1090,44 @@ DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXbeam'] DEBUG - classes: NXinstrument.nxdl.xml:/BEAM NXbeam.nxdl.xml: +NXobject.nxdl.xml: DEBUG - <> DEBUG - documentation (NXinstrument.nxdl.xml:/BEAM): DEBUG - DEBUG - documentation (NXbeam.nxdl.xml:): DEBUG - - Properties of the neutron or X-ray beam at a given location. - - This group is intended to be referenced - by beamline component groups within the :ref:`NXinstrument` group or by the :ref:`NXsample` group. This group is - especially valuable in storing the results of instrument simulations in which it is useful - to specify the beam profile, time distribution etc. at each beamline component. Otherwise, - its most likely use is in the :ref:`NXsample` group in which it defines the results of the neutron - scattering by the sample, e.g., energy transfer, polarizations. Finally, There are cases where the beam is - considered as a beamline component and this group may be defined as a subgroup directly inside - :ref:`NXinstrument`, in which case it is recommended that the position of the beam is specified by an - :ref:`NXtransformations` group, unless the beam is at the origin (which is the sample). - - Note that incident_wavelength and related fields can be a scalar values or arrays, depending on the use case. - To support these use cases, the explicit dimensionality of these fields is not specified, but it can be inferred - by the presense of and shape of accompanying fields, such as incident_wavelength_weights for a polychromatic beam. - + Properties of the neutron or X-ray beam at a given location. + + This group is intended to be referenced + by beamline component groups within the :ref:`NXinstrument` group or by the :ref:`NXsample` group. This group is + especially valuable in storing the results of instrument simulations in which it is useful + to specify the beam profile, time distribution etc. at each beamline component. Otherwise, + its most likely use is in the :ref:`NXsample` group in which it defines the results of the neutron + scattering by the sample, e.g., energy transfer, polarizations. Finally, There are cases where the beam is + considered as a beamline component and this group may be defined as a subgroup directly inside + :ref:`NXinstrument`, in which case it is recommended that the position of the beam is specified by an + :ref:`NXtransformations` group, unless the beam is at the origin (which is the sample). + + Note that ``incident_wavelength``, ``incident_energy``, and related fields can be a scalar values or arrays, depending on the use case. + To support these use cases, the explicit dimensionality of these fields is not specified, but it can be inferred + by the presence of and shape of accompanying fields, such as incident_wavelength_weights for a polychromatic beam. + +DEBUG - documentation (NXobject.nxdl.xml:): +DEBUG - + This is the base object of NeXus. The groups and fields contained + within this file are allowed to be present in any derived base class. + + If nameType="partial", the placeholders (e.g., FIELDNAME or GROUPNAME) + can be replaced by the name of any object (field or group, + respectively) that exists within the same group. + DEBUG - ===== ATTRS (//entry/instrument/beam_pump_0@NX_class) DEBUG - value: NXbeam DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXbeam'] DEBUG - classes: NXinstrument.nxdl.xml:/BEAM NXbeam.nxdl.xml: +NXobject.nxdl.xml: DEBUG - @NX_class [NX_CHAR] DEBUG - DEBUG - ===== FIELD (//entry/instrument/beam_pump_0/average_power): @@ -1023,7 +1138,7 @@ NXbeam.nxdl.xml:/average_power DEBUG - <> DEBUG - documentation (NXbeam.nxdl.xml:/average_power): DEBUG - - Average power at the diagnostic point + Average power at the at the given location. DEBUG - ===== ATTRS (//entry/instrument/beam_pump_0/average_power@units) DEBUG - value: uW @@ -1048,9 +1163,7 @@ DEBUG - classes: NXbeam.nxdl.xml:/distance DEBUG - <> DEBUG - documentation (NXbeam.nxdl.xml:/distance): -DEBUG - - Distance from sample. Note, it is recommended to use NXtransformations instead. - +DEBUG - Distance from sample. Note, it is recommended to use NXtransformations instead. DEBUG - ===== ATTRS (//entry/instrument/beam_pump_0/distance@units) DEBUG - value: cm DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXbeam', 'NX_FLOAT'] @@ -1065,14 +1178,14 @@ NXbeam.nxdl.xml:/fluence DEBUG - <> DEBUG - documentation (NXbeam.nxdl.xml:/fluence): DEBUG - - Incident fluence at the diagnostic point + Incident energy fluence at the given location. DEBUG - ===== ATTRS (//entry/instrument/beam_pump_0/fluence@units) DEBUG - value: mJ/cm^2 DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXbeam', 'NX_FLOAT'] DEBUG - classes: NXbeam.nxdl.xml:/fluence -DEBUG - NXbeam.nxdl.xml:/fluence@units [NX_ANY] +DEBUG - NXbeam.nxdl.xml:/fluence@units [mJ/cm^2] DEBUG - ===== FIELD (//entry/instrument/beam_pump_0/photon_energy): DEBUG - value: 1.55 DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXbeam'] @@ -1106,7 +1219,7 @@ NXbeam.nxdl.xml:/pulse_duration DEBUG - <> DEBUG - documentation (NXbeam.nxdl.xml:/pulse_duration): DEBUG - - FWHM duration of the pulses at the diagnostic point + FWHM duration of the pulses at the given location. DEBUG - ===== ATTRS (//entry/instrument/beam_pump_0/pulse_duration@units) DEBUG - value: fs @@ -1122,7 +1235,7 @@ NXbeam.nxdl.xml:/pulse_energy DEBUG - <> DEBUG - documentation (NXbeam.nxdl.xml:/pulse_energy): DEBUG - - Energy of a single pulse at the diagnostic point + Energy of a single pulse at the given location. DEBUG - ===== ATTRS (//entry/instrument/beam_pump_0/pulse_energy@units) DEBUG - value: nJ @@ -1165,19 +1278,36 @@ DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXpositioner'] DEBUG - classes: NXinstrument.nxdl.xml:/POSITIONER NXpositioner.nxdl.xml: +NXcomponent.nxdl.xml: +NXobject.nxdl.xml: DEBUG - <> DEBUG - documentation (NXinstrument.nxdl.xml:/POSITIONER): DEBUG - DEBUG - documentation (NXpositioner.nxdl.xml:): DEBUG - - A generic positioner such as a motor or piezo-electric transducer. + A generic positioner such as a motor or piezo-electric transducer. + +DEBUG - documentation (NXcomponent.nxdl.xml:): +DEBUG - + Base class for components of an instrument - real ones or simulated ones. +DEBUG - documentation (NXobject.nxdl.xml:): +DEBUG - + This is the base object of NeXus. The groups and fields contained + within this file are allowed to be present in any derived base class. + + If nameType="partial", the placeholders (e.g., FIELDNAME or GROUPNAME) + can be replaced by the name of any object (field or group, + respectively) that exists within the same group. + DEBUG - ===== ATTRS (//entry/instrument/manipulator@NX_class) DEBUG - value: NXpositioner DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXpositioner'] DEBUG - classes: NXinstrument.nxdl.xml:/POSITIONER NXpositioner.nxdl.xml: +NXcomponent.nxdl.xml: +NXobject.nxdl.xml: DEBUG - @NX_class [NX_CHAR] DEBUG - DEBUG - ===== FIELD (//entry/instrument/manipulator/pos_x1): @@ -1281,6 +1411,8 @@ DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/monochromator NXinstrument.nxdl.xml:/MONOCHROMATOR NXmonochromator.nxdl.xml: +NXcomponent.nxdl.xml: +NXobject.nxdl.xml: DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/monochromator): DEBUG - @@ -1288,18 +1420,32 @@ DEBUG - documentation (NXinstrument.nxdl.xml:/MONOCHROMATOR): DEBUG - DEBUG - documentation (NXmonochromator.nxdl.xml:): DEBUG - - A wavelength defining device. - - This is a base class for everything which - selects a wavelength or energy, be it a - monochromator crystal, a velocity selector, - an undulator or whatever. - - The expected units are: - - * wavelength: angstrom - * energy: eV + A wavelength defining device. + + This is a base class for everything which + selects a wavelength or energy, be it a + monochromator crystal, a velocity selector, + an undulator or whatever. + + The expected units are: + + * wavelength: angstrom + * energy: eV + + +DEBUG - documentation (NXcomponent.nxdl.xml:): +DEBUG - + Base class for components of an instrument - real ones or simulated ones. +DEBUG - documentation (NXobject.nxdl.xml:): +DEBUG - + This is the base object of NeXus. The groups and fields contained + within this file are allowed to be present in any derived base class. + + If nameType="partial", the placeholders (e.g., FIELDNAME or GROUPNAME) + can be replaced by the name of any object (field or group, + respectively) that exists within the same group. + DEBUG - ===== ATTRS (//entry/instrument/monochromator@NX_class) DEBUG - value: NXmonochromator DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXmonochromator'] @@ -1307,6 +1453,8 @@ DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/monochromator NXinstrument.nxdl.xml:/MONOCHROMATOR NXmonochromator.nxdl.xml: +NXcomponent.nxdl.xml: +NXobject.nxdl.xml: DEBUG - @NX_class [NX_CHAR] DEBUG - DEBUG - ===== FIELD (//entry/instrument/monochromator/energy): @@ -1331,9 +1479,7 @@ NXmonochromator.nxdl.xml:/energy_error DEBUG - <> DEBUG - DEPRECATED - see https://github.com/nexusformat/definitions/issues/820 DEBUG - documentation (NXmonochromator.nxdl.xml:/energy_error): -DEBUG - - energy standard deviation - +DEBUG - energy standard deviation DEBUG - ===== ATTRS (//entry/instrument/monochromator/energy_error@units) DEBUG - value: eV DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXmonochromator', 'NX_FLOAT'] @@ -1366,15 +1512,15 @@ DEBUG - classes: NXinstrument.nxdl.xml:/name DEBUG - <> DEBUG - documentation (NXinstrument.nxdl.xml:/name): -DEBUG - - Name of instrument - +DEBUG - Name of instrument DEBUG - ===== GROUP (//entry/instrument/source [NXarpes::/NXentry/NXinstrument/NXsource]): DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource'] DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/SOURCE NXinstrument.nxdl.xml:/SOURCE NXsource.nxdl.xml: +NXcomponent.nxdl.xml: +NXobject.nxdl.xml: DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/SOURCE): DEBUG - @@ -1382,11 +1528,24 @@ DEBUG - documentation (NXinstrument.nxdl.xml:/SOURCE): DEBUG - DEBUG - documentation (NXsource.nxdl.xml:): DEBUG - - Radiation source emitting a beam. - - Examples include particle sources (electrons, neutrons, protons) or sources for electromagnetic radiation (photons). - This base class can also be used to describe neutron or x-ray storage ring/facilities. + Radiation source emitting a beam. + + Examples include particle sources (electrons, neutrons, protons) or sources for electromagnetic radiation (photons). + This base class can also be used to describe neutron or x-ray storage ring/facilities. + +DEBUG - documentation (NXcomponent.nxdl.xml:): +DEBUG - + Base class for components of an instrument - real ones or simulated ones. +DEBUG - documentation (NXobject.nxdl.xml:): +DEBUG - + This is the base object of NeXus. The groups and fields contained + within this file are allowed to be present in any derived base class. + + If nameType="partial", the placeholders (e.g., FIELDNAME or GROUPNAME) + can be replaced by the name of any object (field or group, + respectively) that exists within the same group. + DEBUG - ===== ATTRS (//entry/instrument/source@NX_class) DEBUG - value: NXsource DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource'] @@ -1394,6 +1553,8 @@ DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/SOURCE NXinstrument.nxdl.xml:/SOURCE NXsource.nxdl.xml: +NXcomponent.nxdl.xml: +NXobject.nxdl.xml: DEBUG - @NX_class [NX_CHAR] DEBUG - DEBUG - ===== FIELD (//entry/instrument/source/bunch_distance): @@ -1403,9 +1564,7 @@ DEBUG - classes: NXsource.nxdl.xml:/bunch_distance DEBUG - <> DEBUG - documentation (NXsource.nxdl.xml:/bunch_distance): -DEBUG - - For storage rings, time between bunches - +DEBUG - For storage rings, time between bunches DEBUG - ===== ATTRS (//entry/instrument/source/bunch_distance@units) DEBUG - value: us DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource', 'NX_FLOAT'] @@ -1419,9 +1578,7 @@ DEBUG - classes: NXsource.nxdl.xml:/bunch_length DEBUG - <> DEBUG - documentation (NXsource.nxdl.xml:/bunch_length): -DEBUG - - For storage rings, temporal length of the bunch - +DEBUG - For storage rings, temporal length of the bunch DEBUG - ===== ATTRS (//entry/instrument/source/bunch_length@units) DEBUG - value: fs DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource', 'NX_FLOAT'] @@ -1465,9 +1622,7 @@ DEBUG - classes: NXsource.nxdl.xml:/current DEBUG - <> DEBUG - documentation (NXsource.nxdl.xml:/current): -DEBUG - - Accelerator, X-ray tube, or storage ring current - +DEBUG - Accelerator, X-ray tube, or storage ring current DEBUG - ===== ATTRS (//entry/instrument/source/current@units) DEBUG - value: uA DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource', 'NX_FLOAT'] @@ -1482,10 +1637,10 @@ NXsource.nxdl.xml:/energy DEBUG - <> DEBUG - documentation (NXsource.nxdl.xml:/energy): DEBUG - - Source energy. Typically, this would be the energy of - the emitted beam. For storage rings, this would be - the particle beam energy. - + Source energy. Typically, this would be the energy of + the emitted beam. For storage rings, this would be + the particle beam energy. + DEBUG - ===== ATTRS (//entry/instrument/source/energy@units) DEBUG - value: MeV DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource', 'NX_FLOAT'] @@ -1499,9 +1654,7 @@ DEBUG - classes: NXsource.nxdl.xml:/frequency DEBUG - <> DEBUG - documentation (NXsource.nxdl.xml:/frequency): -DEBUG - - Frequency of pulsed source - +DEBUG - Frequency of pulsed source DEBUG - ===== ATTRS (//entry/instrument/source/frequency@units) DEBUG - value: Hz DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource', 'NX_FLOAT'] @@ -1518,21 +1671,22 @@ DEBUG - enumeration (NXsource.nxdl.xml:/mode): DEBUG - -> Single Bunch DEBUG - -> Multi Bunch DEBUG - documentation (NXsource.nxdl.xml:/mode): -DEBUG - - source operating mode - +DEBUG - source operating mode DEBUG - ===== FIELD (//entry/instrument/source/name): DEBUG - value: FLASH DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource', 'NX_CHAR'] DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/SOURCE/name NXsource.nxdl.xml:/name +NXcomponent.nxdl.xml:/name DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/SOURCE/name): DEBUG - DEBUG - documentation (NXsource.nxdl.xml:/name): +DEBUG - Name of source +DEBUG - documentation (NXcomponent.nxdl.xml:/name): DEBUG - - Name of source + Name of the component. DEBUG - ===== FIELD (//entry/instrument/source/number_of_bunches): DEBUG - value: 500 @@ -1541,9 +1695,7 @@ DEBUG - classes: NXsource.nxdl.xml:/number_of_bunches DEBUG - <> DEBUG - documentation (NXsource.nxdl.xml:/number_of_bunches): -DEBUG - - For storage rings, the number of bunches in use. - +DEBUG - For storage rings, the number of bunches in use. DEBUG - ===== FIELD (//entry/instrument/source/number_of_bursts): DEBUG - value: 1 DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource'] @@ -1571,9 +1723,7 @@ DEBUG - -> proton DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/SOURCE/probe): DEBUG - DEBUG - documentation (NXsource.nxdl.xml:/probe): -DEBUG - - type of radiation probe (pick one from the enumerated list and spell exactly) - +DEBUG - type of radiation probe (pick one from the enumerated list and spell exactly) DEBUG - ===== FIELD (//entry/instrument/source/top_up): DEBUG - value: True DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource', 'NX_BOOLEAN'] @@ -1581,9 +1731,7 @@ DEBUG - classes: NXsource.nxdl.xml:/top_up DEBUG - <> DEBUG - documentation (NXsource.nxdl.xml:/top_up): -DEBUG - - Is the synchrotron operating in top_up mode? - +DEBUG - Is the synchrotron operating in top_up mode? DEBUG - ===== FIELD (//entry/instrument/source/type): DEBUG - value: Free Electron Laser DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource', 'NX_CHAR'] @@ -1606,27 +1754,26 @@ DEBUG - -> Ion Source DEBUG - -> UV Plasma Source DEBUG - -> Metal Jet X-ray DEBUG - -> Laser -DEBUG - -> Dye-Laser +DEBUG - -> Dye Laser DEBUG - -> Broadband Tunable Light Source -DEBUG - -> Halogen lamp +DEBUG - -> Halogen Lamp DEBUG - -> LED -DEBUG - -> Mercury Cadmium Telluride +DEBUG - -> Mercury Cadmium Telluride Lamp DEBUG - -> Deuterium Lamp DEBUG - -> Xenon Lamp DEBUG - -> Globar -DEBUG - -> other DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/SOURCE/type): DEBUG - DEBUG - documentation (NXsource.nxdl.xml:/type): -DEBUG - - type of radiation source (pick one from the enumerated list and spell exactly) - +DEBUG - type of radiation source (pick one from the enumerated list and spell exactly) DEBUG - ===== GROUP (//entry/instrument/source_pump [NXarpes::/NXentry/NXinstrument/NXsource]): DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource'] DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/SOURCE NXinstrument.nxdl.xml:/SOURCE NXsource.nxdl.xml: +NXcomponent.nxdl.xml: +NXobject.nxdl.xml: DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/SOURCE): DEBUG - @@ -1634,11 +1781,24 @@ DEBUG - documentation (NXinstrument.nxdl.xml:/SOURCE): DEBUG - DEBUG - documentation (NXsource.nxdl.xml:): DEBUG - - Radiation source emitting a beam. - - Examples include particle sources (electrons, neutrons, protons) or sources for electromagnetic radiation (photons). - This base class can also be used to describe neutron or x-ray storage ring/facilities. + Radiation source emitting a beam. + + Examples include particle sources (electrons, neutrons, protons) or sources for electromagnetic radiation (photons). + This base class can also be used to describe neutron or x-ray storage ring/facilities. + +DEBUG - documentation (NXcomponent.nxdl.xml:): +DEBUG - + Base class for components of an instrument - real ones or simulated ones. +DEBUG - documentation (NXobject.nxdl.xml:): +DEBUG - + This is the base object of NeXus. The groups and fields contained + within this file are allowed to be present in any derived base class. + + If nameType="partial", the placeholders (e.g., FIELDNAME or GROUPNAME) + can be replaced by the name of any object (field or group, + respectively) that exists within the same group. + DEBUG - ===== ATTRS (//entry/instrument/source_pump@NX_class) DEBUG - value: NXsource DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource'] @@ -1646,6 +1806,8 @@ DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/SOURCE NXinstrument.nxdl.xml:/SOURCE NXsource.nxdl.xml: +NXcomponent.nxdl.xml: +NXobject.nxdl.xml: DEBUG - @NX_class [NX_CHAR] DEBUG - DEBUG - ===== FIELD (//entry/instrument/source_pump/bunch_distance): @@ -1655,9 +1817,7 @@ DEBUG - classes: NXsource.nxdl.xml:/bunch_distance DEBUG - <> DEBUG - documentation (NXsource.nxdl.xml:/bunch_distance): -DEBUG - - For storage rings, time between bunches - +DEBUG - For storage rings, time between bunches DEBUG - ===== ATTRS (//entry/instrument/source_pump/bunch_distance@units) DEBUG - value: us DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource', 'NX_FLOAT'] @@ -1671,9 +1831,7 @@ DEBUG - classes: NXsource.nxdl.xml:/bunch_length DEBUG - <> DEBUG - documentation (NXsource.nxdl.xml:/bunch_length): -DEBUG - - For storage rings, temporal length of the bunch - +DEBUG - For storage rings, temporal length of the bunch DEBUG - ===== ATTRS (//entry/instrument/source_pump/bunch_length@units) DEBUG - value: fs DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource', 'NX_FLOAT'] @@ -1707,9 +1865,7 @@ DEBUG - classes: NXsource.nxdl.xml:/frequency DEBUG - <> DEBUG - documentation (NXsource.nxdl.xml:/frequency): -DEBUG - - Frequency of pulsed source - +DEBUG - Frequency of pulsed source DEBUG - ===== ATTRS (//entry/instrument/source_pump/frequency@units) DEBUG - value: Hz DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource', 'NX_FLOAT'] @@ -1726,21 +1882,22 @@ DEBUG - enumeration (NXsource.nxdl.xml:/mode): DEBUG - -> Single Bunch DEBUG - -> Multi Bunch DEBUG - documentation (NXsource.nxdl.xml:/mode): -DEBUG - - source operating mode - +DEBUG - source operating mode DEBUG - ===== FIELD (//entry/instrument/source_pump/name): DEBUG - value: User Laser @ FLASH DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource', 'NX_CHAR'] DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/SOURCE/name NXsource.nxdl.xml:/name +NXcomponent.nxdl.xml:/name DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/SOURCE/name): DEBUG - DEBUG - documentation (NXsource.nxdl.xml:/name): +DEBUG - Name of source +DEBUG - documentation (NXcomponent.nxdl.xml:/name): DEBUG - - Name of source + Name of the component. DEBUG - ===== FIELD (//entry/instrument/source_pump/number_of_bunches): DEBUG - value: 400 @@ -1749,9 +1906,7 @@ DEBUG - classes: NXsource.nxdl.xml:/number_of_bunches DEBUG - <> DEBUG - documentation (NXsource.nxdl.xml:/number_of_bunches): -DEBUG - - For storage rings, the number of bunches in use. - +DEBUG - For storage rings, the number of bunches in use. DEBUG - ===== FIELD (//entry/instrument/source_pump/number_of_bursts): DEBUG - value: 1 DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource'] @@ -1779,9 +1934,7 @@ DEBUG - -> proton DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/SOURCE/probe): DEBUG - DEBUG - documentation (NXsource.nxdl.xml:/probe): -DEBUG - - type of radiation probe (pick one from the enumerated list and spell exactly) - +DEBUG - type of radiation probe (pick one from the enumerated list and spell exactly) DEBUG - ===== FIELD (//entry/instrument/source_pump/rms_jitter): DEBUG - value: 204.68816194453154 DEBUG - classpath: ['NXentry', 'NXinstrument', 'NXsource'] @@ -1814,21 +1967,18 @@ DEBUG - -> Ion Source DEBUG - -> UV Plasma Source DEBUG - -> Metal Jet X-ray DEBUG - -> Laser -DEBUG - -> Dye-Laser +DEBUG - -> Dye Laser DEBUG - -> Broadband Tunable Light Source -DEBUG - -> Halogen lamp +DEBUG - -> Halogen Lamp DEBUG - -> LED -DEBUG - -> Mercury Cadmium Telluride +DEBUG - -> Mercury Cadmium Telluride Lamp DEBUG - -> Deuterium Lamp DEBUG - -> Xenon Lamp DEBUG - -> Globar -DEBUG - -> other DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/INSTRUMENT/SOURCE/type): DEBUG - DEBUG - documentation (NXsource.nxdl.xml:/type): -DEBUG - - type of radiation source (pick one from the enumerated list and spell exactly) - +DEBUG - type of radiation source (pick one from the enumerated list and spell exactly) DEBUG - ===== FIELD (//entry/instrument/spatial_resolution): DEBUG - value: 500 DEBUG - classpath: ['NXentry', 'NXinstrument'] @@ -1856,15 +2006,15 @@ DEBUG - classes: NXentry.nxdl.xml:/run_cycle DEBUG - <> DEBUG - documentation (NXentry.nxdl.xml:/run_cycle): -DEBUG - - Such as "2007-3". Some user facilities organize their beam time into run cycles. - +DEBUG - Such as "2007-3". Some user facilities organize their beam time into run cycles. DEBUG - ===== GROUP (//entry/sample [NXarpes::/NXentry/NXsample]): DEBUG - classpath: ['NXentry', 'NXsample'] DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/SAMPLE NXentry.nxdl.xml:/SAMPLE NXsample.nxdl.xml: +NXcomponent.nxdl.xml: +NXobject.nxdl.xml: DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/SAMPLE): DEBUG - @@ -1872,12 +2022,25 @@ DEBUG - documentation (NXentry.nxdl.xml:/SAMPLE): DEBUG - DEBUG - documentation (NXsample.nxdl.xml:): DEBUG - - Any information on the sample. - - This could include scanned variables that - are associated with one of the data dimensions, e.g. the magnetic field, or - logged data, e.g. monitored temperature vs elapsed time. + Any information on the sample. + + This could include scanned variables that + are associated with one of the data dimensions, e.g. the magnetic field, or + logged data, e.g. monitored temperature vs elapsed time. + +DEBUG - documentation (NXcomponent.nxdl.xml:): +DEBUG - + Base class for components of an instrument - real ones or simulated ones. +DEBUG - documentation (NXobject.nxdl.xml:): +DEBUG - + This is the base object of NeXus. The groups and fields contained + within this file are allowed to be present in any derived base class. + + If nameType="partial", the placeholders (e.g., FIELDNAME or GROUPNAME) + can be replaced by the name of any object (field or group, + respectively) that exists within the same group. + DEBUG - ===== ATTRS (//entry/sample@NX_class) DEBUG - value: NXsample DEBUG - classpath: ['NXentry', 'NXsample'] @@ -1885,6 +2048,8 @@ DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/SAMPLE NXentry.nxdl.xml:/SAMPLE NXsample.nxdl.xml: +NXcomponent.nxdl.xml: +NXobject.nxdl.xml: DEBUG - @NX_class [NX_CHAR] DEBUG - DEBUG - ===== FIELD (//entry/sample/bias): @@ -1928,14 +2093,15 @@ DEBUG - classpath: ['NXentry', 'NXsample', 'NX_CHAR'] DEBUG - classes: NXarpes.nxdl.xml:/ENTRY/SAMPLE/name NXsample.nxdl.xml:/name +NXcomponent.nxdl.xml:/name DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/SAMPLE/name): -DEBUG - - Descriptive name of sample - +DEBUG - Descriptive name of sample DEBUG - documentation (NXsample.nxdl.xml:/name): +DEBUG - Descriptive name of sample +DEBUG - documentation (NXcomponent.nxdl.xml:/name): DEBUG - - Descriptive name of sample + Name of the component. DEBUG - ===== FIELD (//entry/sample/preparation_method): DEBUG - value: in-vacuum cleave @@ -1949,9 +2115,7 @@ DEBUG - classes: NXsample.nxdl.xml:/pressure DEBUG - <> DEBUG - documentation (NXsample.nxdl.xml:/pressure): -DEBUG - - Applied pressure - +DEBUG - Applied pressure DEBUG - ===== ATTRS (//entry/sample/pressure@units) DEBUG - value: mbar DEBUG - classpath: ['NXentry', 'NXsample', 'NX_FLOAT'] @@ -2001,9 +2165,7 @@ DEBUG - classes: NXsample.nxdl.xml:/thickness DEBUG - <> DEBUG - documentation (NXsample.nxdl.xml:/thickness): -DEBUG - - sample thickness - +DEBUG - sample thickness DEBUG - ===== ATTRS (//entry/sample/thickness@units) DEBUG - value: mm DEBUG - classpath: ['NXentry', 'NXsample', 'NX_FLOAT'] @@ -2025,9 +2187,7 @@ DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/start_time): DEBUG - DEBUG - documentation (NXentry.nxdl.xml:/start_time): -DEBUG - - Starting time of measurement - +DEBUG - Starting time of measurement DEBUG - ===== FIELD (//entry/title): DEBUG - value: Excited-state dynamics of WSe2 in the Valence Band and Core-Levels DEBUG - classpath: ['NXentry', 'NX_CHAR'] @@ -2038,9 +2198,7 @@ DEBUG - <> DEBUG - documentation (NXarpes.nxdl.xml:/ENTRY/title): DEBUG - DEBUG - documentation (NXentry.nxdl.xml:/title): -DEBUG - - Extended title for entry - +DEBUG - Extended title for entry DEBUG - ======================== DEBUG - === Default Plotable === DEBUG - ======================== From 7865b39a67c8a1dd96e1f1b36dfe204388e15ce3 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Mon, 17 Mar 2025 13:52:48 +0100 Subject: [PATCH 29/52] catch another pint error --- src/pynxtools/nomad/schema.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pynxtools/nomad/schema.py b/src/pynxtools/nomad/schema.py index 46a97a396..13def6f36 100644 --- a/src/pynxtools/nomad/schema.py +++ b/src/pynxtools/nomad/schema.py @@ -699,7 +699,10 @@ def __create_field(xml_node: ET.Element, container: Section) -> Quantity: quantity = 1 * ureg(nx_dimensionality) dimensionality = quantity.dimensionality - except pint.errors.UndefinedUnitError as err: + except ( + pint.errors.UndefinedUnitError, + pint.errors.DefinitionSyntaxError, + ) as err: raise NotImplementedError( f"Unit {nx_dimensionality} is not supported for {name}." ) from err From 32e32f9d34eebdb34c5b8447d4f61e3eadf5d596 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Mon, 17 Mar 2025 14:52:20 +0100 Subject: [PATCH 30/52] ignore one test case --- src/pynxtools/nomad/schema.py | 13 +++++++------ tests/nomad/test_parsing.py | 13 +++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/pynxtools/nomad/schema.py b/src/pynxtools/nomad/schema.py index 13def6f36..402865e83 100644 --- a/src/pynxtools/nomad/schema.py +++ b/src/pynxtools/nomad/schema.py @@ -436,12 +436,12 @@ def nxdata_ensure_definition( else: filters = ["DATA", "AXISNAME", "FIELDNAME_errors"] # get the reduced options - newdefintions = {} + newdefinitions = {} for dname, definition in self.m_def.all_aliases: if dname not in filters: - newdefintions[dname] = definition + newdefinitions[dname] = definition # run the query - definition = resolve_variadic_name(newdefintions, def_or_name, hint) + definition = resolve_variadic_name(newdefinitions, def_or_name, hint) return definition return super(current_cls, self)._ensure_definition( def_or_name, @@ -481,9 +481,11 @@ def __get_enumeration(xml_node: ET.Element) -> Tuple[Optional[MEnum], Optional[b return None, None items = enumeration.findall("nx:item", __XML_NAMESPACES) - open = bool(enumeration.attrib["open"]) if "open" in enumeration.attrib else False + open_enum = ( + bool(enumeration.attrib["open"]) if "open" in enumeration.attrib else False + ) - return MEnum([value.attrib["value"] for value in items]), open + return MEnum([value.attrib["value"] for value in items]), open_enum def __add_common_properties(xml_node: ET.Element, definition: Definition): @@ -694,7 +696,6 @@ def __create_field(xml_node: ET.Element, container: Section) -> Quantity: dimensionality = NXUnitSet.mapping.get(nx_dimensionality) if not dimensionality and nx_dimensionality != "NX_ANY": try: - # nx_dimensionality = "some_chasdasdkasdn m" from nomad.units import ureg quantity = 1 * ureg(nx_dimensionality) diff --git a/tests/nomad/test_parsing.py b/tests/nomad/test_parsing.py index e050448a7..a235afbf8 100644 --- a/tests/nomad/test_parsing.py +++ b/tests/nomad/test_parsing.py @@ -57,15 +57,16 @@ def test_nexus_example(): ) # good ENUM - x-ray assert instrument.SOURCE[0].probe__field == "x-ray" - # wrong inherited ENUM - Burst - assert instrument.SOURCE[0].mode__field is None - # wrong inherited ENUM for extended field - 'Free Electron Laser' - assert instrument.SOURCE[0].type__field is None + # wrong inherited ENUM - Burst (accepted for open enum) + assert instrument.SOURCE[0].mode__field == "Burst" + # wrong inherited ENUM for extended field - 'Free Electron Laser' (accepted for open enum) + assert instrument.SOURCE[0].type__field == "Free Electron Laser" data = arpes_obj.ENTRY[0].DATA[0] assert len(data.AXISNAME__field) == 3 # there is still a bug in the variadic name resolution, so skip these - # assert data.delays__field is not None - # assert data.angles__field.check("1/Å") + assert data.delays__field is not None + assert data.angles__field.check("1/Å") + # ToDo: if AXISNAME and DATA can be resolved properly, extend this! # assert data.delays__field.check("fs") # but the following still works assert data.energies__field is not None From 741c4a846eecc36ffb9e5f3f827f6f3561d05629 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Mon, 17 Mar 2025 15:37:26 +0100 Subject: [PATCH 31/52] update defs --- src/pynxtools/definitions | 2 +- src/pynxtools/nexus-version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pynxtools/definitions b/src/pynxtools/definitions index db54d2a03..1790395b5 160000 --- a/src/pynxtools/definitions +++ b/src/pynxtools/definitions @@ -1 +1 @@ -Subproject commit db54d2a0394c91898092b67ec71508e0f805ec57 +Subproject commit 1790395b5b8e305ecc4c53bd5849dff85c7d7649 diff --git a/src/pynxtools/nexus-version.txt b/src/pynxtools/nexus-version.txt index 48133eb9b..517561cb8 100644 --- a/src/pynxtools/nexus-version.txt +++ b/src/pynxtools/nexus-version.txt @@ -1 +1 @@ -v2024.02-1885-gdb54d2a0 \ No newline at end of file +v2024.02-1889-g1790395b \ No newline at end of file From 3d1b37c18d802abcdfa44bde7b7a0b253f80eb27 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Mon, 17 Mar 2025 15:37:37 +0100 Subject: [PATCH 32/52] add a test for unit examples --- tests/dataconverter/test_validation.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index 5f261c6f7..f7ccbd7c6 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -950,6 +950,20 @@ def listify_template(data_dict: Template): ], id="baseclass-field-with-illegal-unit", ), + # This can be re-used later when we have proper unit checking + pytest.param( + alter_dict( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/INSTRUMENT[my_instrument]/MONOCHROMATOR[monochromator]/energy_dispersion", + 0.5, + ), + "/ENTRY[my_entry]/INSTRUMENT[my_instrument]/MONOCHROMATOR[monochromator]/energy_dispersion/@units", + "J/mm", + ), + [], + id="baseclass-unit-example", + ), ], ) def test_validate_data_dict(caplog, data_dict, error_messages, request): From 0d0e7e568fdf460553b060b9560452e3d6802ce5 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Mon, 17 Mar 2025 16:51:23 +0100 Subject: [PATCH 33/52] only give UnitWithoutDocumentation if not ignore_undocumented --- src/pynxtools/dataconverter/validation.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index 7b61446d0..463eb038f 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -812,11 +812,12 @@ def startswith_with_variations( # check that parent has units node = add_best_matches_for(not_visited_key.rsplit("/", 1)[0], tree) if node is None or node.type != "field" or node.unit is None: - collector.collect_and_log( - not_visited_key, - ValidationProblem.UnitWithoutDocumentation, - mapping[not_visited_key], - ) + if not ignore_undocumented: + collector.collect_and_log( + not_visited_key, + ValidationProblem.UnitWithoutDocumentation, + mapping[not_visited_key], + ) # parent key will be checked on its own if it exists, because it is in the list continue From 32779741b03c4216c4ba289dbfe248e7a78bd909 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Tue, 18 Mar 2025 18:06:03 +0100 Subject: [PATCH 34/52] add a test for variadic concept for required field --- src/pynxtools/data/NXtest.nxdl.xml | 14 +++++ tests/dataconverter/test_validation.py | 86 +++++++++++++++++++++----- 2 files changed, 83 insertions(+), 17 deletions(-) diff --git a/src/pynxtools/data/NXtest.nxdl.xml b/src/pynxtools/data/NXtest.nxdl.xml index c6ae39bc5..99a263271 100644 --- a/src/pynxtools/data/NXtest.nxdl.xml +++ b/src/pynxtools/data/NXtest.nxdl.xml @@ -27,6 +27,20 @@ A dummy entry to test optional parent check for an optional child. + + A group with a (specified) name, but nameType not given explicitly. + + + + + + + A group with a name and nameType="specified". + + + + + diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index f7ccbd7c6..3d468fb2d 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -84,6 +84,29 @@ def listify_template(data_dict: Template): TEMPLATE = Template() +TEMPLATE["required"]["/ENTRY[my_entry]/definition"] = "NXtest" # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/definition/@version"] = "2.4.6" # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/program_name"] = "Testing program" # pylint: disable=E1126 + +TEMPLATE["required"]["/ENTRY[my_entry]/OPTIONAL_group[my_group]/required_field"] = 1 +TEMPLATE["optional"]["/ENTRY[my_entry]/OPTIONAL_group[my_group]/optional_field"] = 1 + +TEMPLATE["required"][ + "/ENTRY[my_entry]/specified_group_with_no_name_type/specified_field_with_no_name_type" +] = 1.0 +TEMPLATE["required"][ + "/ENTRY[my_entry]/specified_group_with_no_name_type/specified_field_with_no_name_type/@specified_attr_in_field_with_no_name_type" +] = "data" +TEMPLATE["required"][ + "/ENTRY[my_entry]/specified_group_with_no_name_type/@specified_attr_with_no_name_type" +] = "attr" + +TEMPLATE["required"]["/ENTRY[my_entry]/specified_group/specified_field"] = 1.0 +TEMPLATE["required"][ + "/ENTRY[my_entry]/specified_group/specified_field/@specified_attr_in_field" +] = "attr" +TEMPLATE["required"]["/ENTRY[my_entry]/specified_group/@specified_attr"] = "attr" + TEMPLATE["optional"][ "/ENTRY[my_entry]/NXODD_name[nxodd_name]/anamethatRENAMES[anamethatichangetothis]" ] = 2 @@ -94,16 +117,15 @@ def listify_template(data_dict: Template): TEMPLATE["optional"][ "/ENTRY[my_entry]/NXODD_name[nxodd_name]/DATA[float_value_no_attr]" ] = (2.0,) -TEMPLATE["optional"]["/ENTRY[my_entry]/optional_parent/required_child"] = 1 # pylint: disable=E1126 -TEMPLATE["optional"]["/ENTRY[my_entry]/optional_parent/optional_child"] = 1 # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value"] = True # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value/@units"] = "" -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value"] = 2 # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value/@units"] = "eV" # pylint: disable=E1126 TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/number_value"] = 2 TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/number_value/@units"] = ( "eV" ) +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value"] = True # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/bool_value/@units"] = "" +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value"] = 2 # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/int_value/@units"] = "eV" # pylint: disable=E1126 + TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/posint_value"] = np.array( [1, 2, 3], # pylint: disable=E1126 dtype=np.int8, @@ -118,8 +140,15 @@ def listify_template(data_dict: Template): TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/@group_attribute"] = ( "data" # pylint: disable=E1126 ) +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value"] = ( + "2022-01-22T12:14:12.05018+00:00" # pylint: disable=E1126 +) +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value/@units"] = "" +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/type"] = "2nd type" # pylint: disable=E1126 +TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/type/@array"] = [0, 1, 2] TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/@signal"] = "data" TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/DATA[data]"] = 1 # pylint: disable=E1126 + TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/bool_value"] = True # pylint: disable=E1126 TEMPLATE["required"][ "/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/bool_value/@units" @@ -163,23 +192,16 @@ def listify_template(data_dict: Template): ) TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/@signal"] = "data" TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_two_name]/DATA[data]"] = 1 # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/OPTIONAL_group[my_group]/required_field"] = 1 # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/definition"] = "NXtest" # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/definition/@version"] = "2.4.6" # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/program_name"] = "Testing program" # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/type"] = "2nd type" # pylint: disable=E1126 -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/type/@array"] = [0, 1, 2] -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value"] = ( - "2022-01-22T12:14:12.05018+00:00" # pylint: disable=E1126 -) -TEMPLATE["required"]["/ENTRY[my_entry]/NXODD_name[nxodd_name]/date_value/@units"] = "" -TEMPLATE["optional"]["/ENTRY[my_entry]/OPTIONAL_group[my_group]/optional_field"] = 1 + TEMPLATE["optional"]["/ENTRY[my_entry]/required_group/description"] = ( "An example description" ) TEMPLATE["optional"]["/ENTRY[my_entry]/required_group2/description"] = ( "An example description" ) + +TEMPLATE["required"]["/ENTRY[my_entry]/optional_parent/required_child"] = 1 # pylint: disable=E1126 +TEMPLATE["optional"]["/ENTRY[my_entry]/optional_parent/optional_child"] = 1 # pylint: disable=E1126 TEMPLATE["required"][ "/ENTRY[my_entry]/optional_parent/req_group_in_opt_group/DATA[data]" ] = 1 @@ -657,6 +679,36 @@ def listify_template(data_dict: Template): [], id="no-child-provided-optional-parent", ), + pytest.param( + alter_dict( + remove_from_dict( + TEMPLATE, + "/ENTRY[my_entry]/optional_parent/required_child", + "required", + ), + "/ENTRY[my_entry]/optional_parent/DATA[required_child]", + 1, + ), + [], + id="concept-name-given-for-nonvariadic-field", + ), + pytest.param( + alter_dict( + remove_from_dict( + TEMPLATE, + "/ENTRY[my_entry]/optional_parent/optional_child", + "optional", + ), + "/ENTRY[my_entry]/optional_parent/AXISNAME[optional_child]", + "test value", + ), + [ + "The value at /ENTRY[my_entry]/optional_parent/AXISNAME[optional_child] should be " + "one of the following Python types: (, ), as " + "defined in the NXDL as NX_INT." + ], + id="concept-name-given-for-nonvariadic-field-wrong-type", + ), pytest.param(TEMPLATE, "", id="valid-data-dict"), pytest.param( remove_from_dict(TEMPLATE, "/ENTRY[my_entry]/required_group/description"), From ce9669014347212f97e90017495ca335b2b6639d Mon Sep 17 00:00:00 2001 From: sanbrock Date: Tue, 18 Mar 2025 21:20:35 +0100 Subject: [PATCH 35/52] adding tests for checking attributes --- tests/nomad/test_parsing.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/nomad/test_parsing.py b/tests/nomad/test_parsing.py index a235afbf8..694517c98 100644 --- a/tests/nomad/test_parsing.py +++ b/tests/nomad/test_parsing.py @@ -77,6 +77,16 @@ def test_nexus_example(): assert (1 * data.AXISNAME__field["angles__field"].unit).check("1/Å") assert (1 * data.AXISNAME__field["delays__field"].unit).check("fs") assert data.___axes == "['angles', 'energies', 'delays']" + # testing attributes + assert ( + data.AXISNAME__field["angles__field"].attributes.get("m_nx_data_path") + == "/entry/data/angles" + ) + assert ( + data.m_get_quantity_attribute("angles__field", "m_nx_data_path") + == "/entry/data/angles" + ) + assert data.m_attributes.get("m_nx_data_path") == "/entry/data" def test_same_name_field_and_group(): From 4753cb2b9eefebf432a54bdc43f4b105620173a2 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Wed, 19 Mar 2025 10:19:38 +0100 Subject: [PATCH 36/52] add test for nameType=any --- src/pynxtools/data/NXtest.nxdl.xml | 7 ++++++ tests/dataconverter/test_validation.py | 31 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/src/pynxtools/data/NXtest.nxdl.xml b/src/pynxtools/data/NXtest.nxdl.xml index 99a263271..f40229aaa 100644 --- a/src/pynxtools/data/NXtest.nxdl.xml +++ b/src/pynxtools/data/NXtest.nxdl.xml @@ -41,6 +41,13 @@ + + A group with a name and nameType="any". + + + + + diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index 3d468fb2d..31de55d00 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -107,6 +107,16 @@ def listify_template(data_dict: Template): ] = "attr" TEMPLATE["required"]["/ENTRY[my_entry]/specified_group/@specified_attr"] = "attr" +TEMPLATE["required"][ + "/ENTRY[my_entry]/any_groupGROUP[anyany_groupGROUP]/any_fieldFIELD[any_fieldFIELD]" +] = 1.0 +TEMPLATE["required"][ + "/ENTRY[my_entry]/any_groupGROUP[anyany_groupGROUP]/any_fieldFIELD[any_fieldFIELD]/any_attrATTR_in_field[any_attrATTR_in_field]" +] = "attr" +TEMPLATE["required"][ + "/ENTRY[my_entry]/any_groupGROUP[anyany_groupGROUP]/any_fieldFIELD[any_fieldFIELD]/any_attrATTR[any_attrATTR]" +] = "attr" + TEMPLATE["optional"][ "/ENTRY[my_entry]/NXODD_name[nxodd_name]/anamethatRENAMES[anamethatichangetothis]" ] = 2 @@ -223,6 +233,27 @@ def listify_template(data_dict: Template): @pytest.mark.parametrize( "data_dict,error_messages", [ + pytest.param( + alter_dict( + alter_dict( + remove_from_dict( + remove_from_dict( + TEMPLATE, + "/ENTRY[my_entry]/any_groupGROUP[anyany_groupGROUP]/any_fieldFIELD[any_fieldFIELD]/any_attrATTR_in_field[any_attrATTR_in_field]", + "required", + ), + "/ENTRY[my_entry]/any_groupGROUP[anyany_groupGROUP]/any_attrATTR[any_attrATTR]", + "required", + ), + "/ENTRY[my_entry]/any_groupGROUP[some_group_name]/any_fieldFIELD[some_field_name]/any_attrATTR_in_field[some_attr_name]", + "new attr", + ), + "/ENTRY[my_entry]/any_groupGROUP[some_group_name]/any_attrATTR[some_attr_name]", + "new attr", + ), + [], + id="name-type-any", + ), pytest.param( alter_dict( TEMPLATE, From 95446519b817c70a0078a96ad94752fcd09dc17e Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Wed, 19 Mar 2025 12:00:11 +0100 Subject: [PATCH 37/52] properly handle variadic attributes in groups and fields --- src/pynxtools/data/NXtest.nxdl.xml | 6 ++--- src/pynxtools/dataconverter/validation.py | 22 ++++++++++++++--- tests/dataconverter/test_validation.py | 29 +++++++++++++++-------- 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/src/pynxtools/data/NXtest.nxdl.xml b/src/pynxtools/data/NXtest.nxdl.xml index f40229aaa..b3b81a61c 100644 --- a/src/pynxtools/data/NXtest.nxdl.xml +++ b/src/pynxtools/data/NXtest.nxdl.xml @@ -43,10 +43,10 @@ A group with a name and nameType="any". - - + + - + diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index 463eb038f..a5cc7a974 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -112,9 +112,9 @@ def default_to_regular_dict(d): def split_class_and_name_of(name: str) -> Tuple[Optional[str], str]: """ Return the class and the name of a data dict entry of the form - `get_class_of("ENTRY[entry]")`, which will return `("ENTRY", "entry")`. + `split_class_and_name_of("ENTRY[entry]")`, which will return `("ENTRY", "entry")`. If this is a simple string it will just return this string, i.e. - `get_class_of("entry")` will return `None, "entry"`. + `split_class_and_name_of("entry")` will return `None, "entry"`. Args: name (str): The data dict entry @@ -204,11 +204,19 @@ def get_variations_of(node: NexusNode, keys: Mapping[str, Any]) -> List[str]: return [f"{convert_nexus_to_caps(node.nx_class)}[{node.name}]"] variations = [] + for key in keys: concept_name, instance_name = split_class_and_name_of(key) if node.type == "attribute": # Remove the starting @ from attributes + if concept_name: + concept_name = ( + concept_name[1:] + if concept_name.startswith("@") + else concept_name + ) + instance_name = ( instance_name[1:] if instance_name.startswith("@") @@ -217,6 +225,7 @@ def get_variations_of(node: NexusNode, keys: Mapping[str, Any]) -> List[str]: if not concept_name or concept_name != node.name: continue + name_any = node.name_type == "any" name_partial = node.name_type == "partial" @@ -233,8 +242,12 @@ def get_variations_of(node: NexusNode, keys: Mapping[str, Any]) -> List[str]: return variations def get_field_attributes(name: str, keys: Mapping[str, Any]) -> Mapping[str, Any]: + prefix = f"{name}@" return { - f"@{k.split('@')[1]}": keys[k] for k in keys if k.startswith(f"{name}@") + # Preserve everything after the field name, keeping '@attr[@attr]' or '@attr' + k[len(prefix) - 1 :]: v + for k, v in keys.items() + if k.startswith(prefix) } def handle_nxdata(node: NexusGroup, keys: Mapping[str, Any], prev_path: str): @@ -385,6 +398,8 @@ def handle_group(node: NexusGroup, keys: Mapping[str, Any], prev_path: str): continue if node.nx_class == "NXdata": handle_nxdata(node, keys[variant], prev_path=f"{prev_path}/{variant}") + # if node.nx_class == "NXcollection": + # ToDo: stop recursion here else: recurse_tree(node, keys[variant], prev_path=f"{prev_path}/{variant}") @@ -481,6 +496,7 @@ def handle_field(node: NexusNode, keys: Mapping[str, Any], prev_path: str): def handle_attribute(node: NexusNode, keys: Mapping[str, Any], prev_path: str): full_path = remove_from_not_visited(f"{prev_path}/@{node.name}") variants = get_variations_of(node, keys) + if ( not variants and node.optionality == "required" diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index 31de55d00..8e0eeb981 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -107,14 +107,15 @@ def listify_template(data_dict: Template): ] = "attr" TEMPLATE["required"]["/ENTRY[my_entry]/specified_group/@specified_attr"] = "attr" + TEMPLATE["required"][ - "/ENTRY[my_entry]/any_groupGROUP[anyany_groupGROUP]/any_fieldFIELD[any_fieldFIELD]" + "/ENTRY[my_entry]/any_groupGROUP[any_groupGROUP]/any_fieldFIELD[any_fieldFIELD]" ] = 1.0 TEMPLATE["required"][ - "/ENTRY[my_entry]/any_groupGROUP[anyany_groupGROUP]/any_fieldFIELD[any_fieldFIELD]/any_attrATTR_in_field[any_attrATTR_in_field]" + "/ENTRY[my_entry]/any_groupGROUP[any_groupGROUP]/any_fieldFIELD[any_fieldFIELD]/@any_attrATTR_in_field[@any_attrATTR_in_field]" ] = "attr" TEMPLATE["required"][ - "/ENTRY[my_entry]/any_groupGROUP[anyany_groupGROUP]/any_fieldFIELD[any_fieldFIELD]/any_attrATTR[any_attrATTR]" + "/ENTRY[my_entry]/any_groupGROUP[any_groupGROUP]/@any_attrATTR[@any_attrATTR]" ] = "attr" TEMPLATE["optional"][ @@ -236,19 +237,27 @@ def listify_template(data_dict: Template): pytest.param( alter_dict( alter_dict( - remove_from_dict( + alter_dict( remove_from_dict( - TEMPLATE, - "/ENTRY[my_entry]/any_groupGROUP[anyany_groupGROUP]/any_fieldFIELD[any_fieldFIELD]/any_attrATTR_in_field[any_attrATTR_in_field]", + remove_from_dict( + remove_from_dict( + TEMPLATE, + "/ENTRY[my_entry]/any_groupGROUP[any_groupGROUP]/any_fieldFIELD[any_fieldFIELD]", + "required", + ), + "/ENTRY[my_entry]/any_groupGROUP[any_groupGROUP]/any_fieldFIELD[any_fieldFIELD]/@any_attrATTR_in_field[@any_attrATTR_in_field]", + "required", + ), + "/ENTRY[my_entry]/any_groupGROUP[any_groupGROUP]/@any_attrATTR[@any_attrATTR]", "required", ), - "/ENTRY[my_entry]/any_groupGROUP[anyany_groupGROUP]/any_attrATTR[any_attrATTR]", - "required", + "/ENTRY[my_entry]/any_groupGROUP[some_group_name]/any_fieldFIELD[some_field_name]", + 1.0, ), - "/ENTRY[my_entry]/any_groupGROUP[some_group_name]/any_fieldFIELD[some_field_name]/any_attrATTR_in_field[some_attr_name]", + "/ENTRY[my_entry]/any_groupGROUP[some_group_name]/any_fieldFIELD[some_field_name]/@any_attrATTR_in_field[@some_attr_name]", "new attr", ), - "/ENTRY[my_entry]/any_groupGROUP[some_group_name]/any_attrATTR[some_attr_name]", + "/ENTRY[my_entry]/any_groupGROUP[some_group_name]/@any_attrATTR[@some_attr_name]", "new attr", ), [], From 0b73752a4d4ca5d6d52054e6503bf878b3ebc403 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Wed, 19 Mar 2025 12:16:16 +0100 Subject: [PATCH 38/52] fix reader test --- src/pynxtools/dataconverter/readers/example/reader.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/pynxtools/dataconverter/readers/example/reader.py b/src/pynxtools/dataconverter/readers/example/reader.py index 7e368a264..2abbaa97d 100644 --- a/src/pynxtools/dataconverter/readers/example/reader.py +++ b/src/pynxtools/dataconverter/readers/example/reader.py @@ -57,7 +57,13 @@ def read( # The entries in the template dict should correspond with what the dataconverter # outputs with --generate-template for a provided NXDL file if ( - k.startswith("/ENTRY[entry]/required_group") + k.startswith( + ( + "/ENTRY[entry]/required_group", + "/ENTRY[entry]/specified_group", + "/ENTRY[entry]/any_groupGROUP[any_groupgroup]", + ) + ) or k in ( "/ENTRY[entry]/optional_parent/req_group_in_opt_group", From c211caf1105a04acbf2e225b4b8c0a36b58b6e57 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Wed, 19 Mar 2025 14:19:47 +0100 Subject: [PATCH 39/52] update test framework for plugins to allow skipping expected output --- src/pynxtools/testing/nexus_conversion.py | 60 ++++++++++++++--------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/src/pynxtools/testing/nexus_conversion.py b/src/pynxtools/testing/nexus_conversion.py index 32c5afb0f..04016ee23 100644 --- a/src/pynxtools/testing/nexus_conversion.py +++ b/src/pynxtools/testing/nexus_conversion.py @@ -20,7 +20,7 @@ import logging import os from glob import glob -from typing import Dict, List, Literal, Tuple +from typing import Dict, List, Literal, Tuple, Optional try: from nomad.client import parse @@ -30,13 +30,11 @@ NOMAD_AVAILABLE = False -from pynxtools.dataconverter.convert import get_reader, transfer_data_into_template +from pynxtools.dataconverter.convert import get_reader, convert from pynxtools.dataconverter.helpers import ( add_default_root_attributes, get_nxdl_root_and_path, ) -from pynxtools.dataconverter.validation import validate_dict_against -from pynxtools.dataconverter.writer import Writer from pynxtools.nexus.nexus import HandleNexus @@ -115,7 +113,11 @@ def convert_to_nexus( self.ref_nexus_file = [file for file in example_files if file.endswith(".nxs")][ 0 ] - input_files = [file for file in example_files if not file.endswith(".nxs")] + input_files = [ + file + for file in example_files + if not file.endswith((".nxs", "ref_output.txt")) + ] assert self.ref_nexus_file, "Reference nexus (.nxs) file not found" assert ( @@ -126,28 +128,40 @@ def convert_to_nexus( nxdl_root, nxdl_file = get_nxdl_root_and_path(self.nxdl) assert os.path.exists(nxdl_file), f"NXDL file {nxdl_file} not found" - read_data = transfer_data_into_template( - input_file=input_files, - reader=self.reader_name, - nxdl_name=self.nxdl, - nxdl_root=nxdl_root, - skip_verify=True, - **self.kwargs, - ) - - # Clear the log of `transfer_data_into_template` + # Clear the log of `convert` self.caplog.clear() with self.caplog.at_level(caplog_level): - _ = validate_dict_against( - self.nxdl, read_data, ignore_undocumented=ignore_undocumented - )[0] - assert self.caplog.text == "" + _ = convert( + input_file=input_files, + reader=self.reader_name, + nxdl=self.nxdl, + skip_verify=False, + ignore_undocumented=ignore_undocumented, + output=self.created_nexus, + **self.kwargs, + ) - add_default_root_attributes( - data=read_data, filename=os.path.basename(self.created_nexus) - ) - Writer(read_data, nxdl_file, self.created_nexus).write() + test_output = self.caplog.messages + + files_with_expected_output = [ + file for file in example_files if file.endswith("ref_output.txt") + ] + + if files_with_expected_output: + output_file = files_with_expected_output[0] + with open(output_file, "r") as file: + expected_messages = [line.strip() for line in file.readlines()] + + for message in expected_messages: + if caplog_level == "WARNING": + if message.startswith(("WARNING", "ERROR")): + test_output.remove(message) + if caplog_level == "ERROR": + if message.startswith("ERROR"): + test_output.remove(message) + + assert test_output == [] if NOMAD_AVAILABLE: kwargs = dict( From feec5dff313b4c65a189138fdb04160eae220087 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Wed, 19 Mar 2025 14:20:50 +0100 Subject: [PATCH 40/52] add tests for identifierNAME --- tests/dataconverter/test_validation.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index 8e0eeb981..124af4f7c 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -1042,6 +1042,28 @@ def listify_template(data_dict: Template): ], id="baseclass-field-with-illegal-unit", ), + pytest.param( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/identifierNAME[identifier_id]", + "123", + ), + [], + id="identifier", + ), + pytest.param( + alter_dict( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/identifierNAME[identifier_id]", + "123", + ), + "/ENTRY[my_entry]/identifierNAME[identifier_id]/@type", + "ORCID", + ), + [], + id="identifier-with-type", + ), # This can be re-used later when we have proper unit checking pytest.param( alter_dict( From 7b9546ebf6b07a9f60c12a3d341f19b404fa99e7 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Wed, 19 Mar 2025 16:17:48 +0100 Subject: [PATCH 41/52] handling of identifierNAME and NXcollection --- src/pynxtools/data/NXtest.nxdl.xml | 4 ++ .../dataconverter/readers/example/reader.py | 2 + src/pynxtools/dataconverter/validation.py | 13 +++- tests/dataconverter/test_validation.py | 68 +++++++++++++++++-- 4 files changed, 79 insertions(+), 8 deletions(-) diff --git a/src/pynxtools/data/NXtest.nxdl.xml b/src/pynxtools/data/NXtest.nxdl.xml index b3b81a61c..b44330d21 100644 --- a/src/pynxtools/data/NXtest.nxdl.xml +++ b/src/pynxtools/data/NXtest.nxdl.xml @@ -121,5 +121,9 @@ A required NXuser entry. + + + + diff --git a/src/pynxtools/dataconverter/readers/example/reader.py b/src/pynxtools/dataconverter/readers/example/reader.py index 2abbaa97d..976bd89bf 100644 --- a/src/pynxtools/dataconverter/readers/example/reader.py +++ b/src/pynxtools/dataconverter/readers/example/reader.py @@ -62,6 +62,8 @@ def read( "/ENTRY[entry]/required_group", "/ENTRY[entry]/specified_group", "/ENTRY[entry]/any_groupGROUP[any_groupgroup]", + "/ENTRY[entry]/identified_user", + "/ENTRY[entry]/named_collection", ) ) or k diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index a5cc7a974..e073aa099 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -398,8 +398,8 @@ def handle_group(node: NexusGroup, keys: Mapping[str, Any], prev_path: str): continue if node.nx_class == "NXdata": handle_nxdata(node, keys[variant], prev_path=f"{prev_path}/{variant}") - # if node.nx_class == "NXcollection": - # ToDo: stop recursion here + if node.nx_class == "NXcollection": + return else: recurse_tree(node, keys[variant], prev_path=f"{prev_path}/{variant}") @@ -557,6 +557,10 @@ def add_best_matches_for(key: str, node: NexusNode) -> Optional[NexusNode]: if node is None: return None + if node.type == "group" and node.nx_class == "NXcollection": + # Stop iterating, return NXcollection node + return node + return node def is_documented(key: str, tree: NexusNode) -> bool: @@ -565,9 +569,14 @@ def is_documented(key: str, tree: NexusNode) -> bool: return True node = add_best_matches_for(key, tree) + if node is None: return False + if node.type == "group" and node.nx_class == "NXcollection": + # Collection found, mark as documented + return True + if isinstance(mapping[key], dict) and "link" in mapping[key]: # TODO: Follow link and check consistency with current field return True diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index 124af4f7c..cd26d76cc 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -678,7 +678,7 @@ def listify_template(data_dict: Template): ), pytest.param( set_to_none_in_dict( - TEMPLATE, "/ENTRY[my_entry]/optional_parent/required_child", "optional" + TEMPLATE, "/ENTRY[my_entry]/optional_parent/required_child", "required" ), [ "The data entry corresponding to /ENTRY[my_entry]/optional_parent/" @@ -726,10 +726,13 @@ def listify_template(data_dict: Template): "/ENTRY[my_entry]/optional_parent/required_child", "required", ), - "/ENTRY[my_entry]/optional_parent/DATA[required_child]", + "/ENTRY[my_entry]/optional_parent/AXISNAME[required_child]", 1, ), - [], + [ + "The data entry corresponding to /ENTRY[my_entry]/optional_parent/" + "required_child is required and hasn't been supplied by the reader." + ], id="concept-name-given-for-nonvariadic-field", ), pytest.param( @@ -1045,11 +1048,38 @@ def listify_template(data_dict: Template): pytest.param( alter_dict( TEMPLATE, - "/ENTRY[my_entry]/identifierNAME[identifier_id]", + "/ENTRY[my_entry]/identified_user/identifier_1", "123", ), [], - id="identifier", + id="specified-identifier-with-type", + ), + # ToDo: reactivate if sibling inheritance works properly + # pytest.param( + # alter_dict( + # alter_dict( + # TEMPLATE, + # "/ENTRY[my_entry]/identified_user/identifier_1", + # "123", + # ), + # "/ENTRY[my_entry]/identified_user/identifier_1/@type", + # "ORCID", + # ), + # [], + # id="specified-identifier-with-type", + # ), + pytest.param( + alter_dict( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/identifierNAME[identifier_id]", + "123", + ), + "/ENTRY[my_entry]/identifierNAME[identifier_id]/@type", + "ORCID", + ), + [], + id="name-fitted-identifier-with-type", ), pytest.param( alter_dict( @@ -1062,7 +1092,7 @@ def listify_template(data_dict: Template): "ORCID", ), [], - id="identifier-with-type", + id="name-fitted-identifier-with-type", ), # This can be re-used later when we have proper unit checking pytest.param( @@ -1078,6 +1108,32 @@ def listify_template(data_dict: Template): [], id="baseclass-unit-example", ), + pytest.param( + alter_dict( + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/COLLECTION[collection]/some_field", + 0.5, + ), + "/ENTRY[my_entry]/COLLECTION[collection]/DATA[data]/some_field", + 0.5, + ), + [], + id="variadic-nxcollection", + ), + pytest.param( + alter_dict( + alter_dict( + TEMPLATE, + "/ENTRY[entry]/named_collection/some_field", + 0.5, + ), + "/ENTRY[entry]/named_collection/DATA[data]/some_field", + 0.5, + ), + [], + id="nonvariadic-nxcollection", + ), ], ) def test_validate_data_dict(caplog, data_dict, error_messages, request): From 53bdacb7da23ab163696d99d25cd5a28cc12daaa Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Wed, 19 Mar 2025 17:59:26 +0100 Subject: [PATCH 42/52] correctly check for named collections --- src/pynxtools/data/NXtest.nxdl.xml | 4 ++-- .../dataconverter/readers/example/reader.py | 2 +- src/pynxtools/dataconverter/validation.py | 17 +++++++++++++---- tests/dataconverter/test_validation.py | 11 ++++++----- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/src/pynxtools/data/NXtest.nxdl.xml b/src/pynxtools/data/NXtest.nxdl.xml index b44330d21..329f42ef2 100644 --- a/src/pynxtools/data/NXtest.nxdl.xml +++ b/src/pynxtools/data/NXtest.nxdl.xml @@ -121,9 +121,9 @@ A required NXuser entry. - + - + diff --git a/src/pynxtools/dataconverter/readers/example/reader.py b/src/pynxtools/dataconverter/readers/example/reader.py index 976bd89bf..0d60b335e 100644 --- a/src/pynxtools/dataconverter/readers/example/reader.py +++ b/src/pynxtools/dataconverter/readers/example/reader.py @@ -62,7 +62,7 @@ def read( "/ENTRY[entry]/required_group", "/ENTRY[entry]/specified_group", "/ENTRY[entry]/any_groupGROUP[any_groupgroup]", - "/ENTRY[entry]/identified_user", + "/ENTRY[entry]/identified_calibration", "/ENTRY[entry]/named_collection", ) ) diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index e073aa099..3e84f93fb 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -368,6 +368,7 @@ def check_nxdata(): def handle_group(node: NexusGroup, keys: Mapping[str, Any], prev_path: str): variants = get_variations_of(node, keys) + if node.parent_of: for child in node.parent_of: variants += get_variations_of(child, keys) @@ -557,10 +558,6 @@ def add_best_matches_for(key: str, node: NexusNode) -> Optional[NexusNode]: if node is None: return None - if node.type == "group" and node.nx_class == "NXcollection": - # Stop iterating, return NXcollection node - return node - return node def is_documented(key: str, tree: NexusNode) -> bool: @@ -571,6 +568,18 @@ def is_documented(key: str, tree: NexusNode) -> bool: node = add_best_matches_for(key, tree) if node is None: + key_path = key.replace("@", "") + while "/" in key_path: + key_path = key_path.rsplit("/", 1)[0] # Remove last segment + parent_node = add_best_matches_for(key_path, tree) + if ( + parent_node + and parent_node.type == "group" + and parent_node.nx_class == "NXcollection" + ): + # Collection found for parents, mark as documented + return True + return False if node.type == "group" and node.nx_class == "NXcollection": diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index cd26d76cc..39e0c2d6c 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -729,6 +729,7 @@ def listify_template(data_dict: Template): "/ENTRY[my_entry]/optional_parent/AXISNAME[required_child]", 1, ), + # ToDo: should not raise a warning if sibling inheritance works [ "The data entry corresponding to /ENTRY[my_entry]/optional_parent/" "required_child is required and hasn't been supplied by the reader." @@ -1048,7 +1049,7 @@ def listify_template(data_dict: Template): pytest.param( alter_dict( TEMPLATE, - "/ENTRY[my_entry]/identified_user/identifier_1", + "/ENTRY[my_entry]/identified_calibration/identifier_1", "123", ), [], @@ -1059,10 +1060,10 @@ def listify_template(data_dict: Template): # alter_dict( # alter_dict( # TEMPLATE, - # "/ENTRY[my_entry]/identified_user/identifier_1", + # "/ENTRY[my_entry]/identified_calibration/identifier_1", # "123", # ), - # "/ENTRY[my_entry]/identified_user/identifier_1/@type", + # "/ENTRY[my_entry]/identified_calibration/identifier_1/@type", # "ORCID", # ), # [], @@ -1125,10 +1126,10 @@ def listify_template(data_dict: Template): alter_dict( alter_dict( TEMPLATE, - "/ENTRY[entry]/named_collection/some_field", + "/ENTRY[my_entry]/named_collection/some_field", 0.5, ), - "/ENTRY[entry]/named_collection/DATA[data]/some_field", + "/ENTRY[my_entry]/named_collection/DATA[data]/some_field", 0.5, ), [], From 05ce29f5b89d3747919300f2dc81da1bf60b590f Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Wed, 19 Mar 2025 18:31:11 +0100 Subject: [PATCH 43/52] linting fix --- src/pynxtools/testing/nexus_conversion.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pynxtools/testing/nexus_conversion.py b/src/pynxtools/testing/nexus_conversion.py index 04016ee23..01ccbd849 100644 --- a/src/pynxtools/testing/nexus_conversion.py +++ b/src/pynxtools/testing/nexus_conversion.py @@ -133,7 +133,7 @@ def convert_to_nexus( with self.caplog.at_level(caplog_level): _ = convert( - input_file=input_files, + input_file=tuple(input_files), reader=self.reader_name, nxdl=self.nxdl, skip_verify=False, From 3eba3156b44ad80327f767fda20621d2421b7398 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Wed, 19 Mar 2025 18:48:02 +0100 Subject: [PATCH 44/52] remove commented code --- src/pynxtools/dataconverter/nexus_tree.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/pynxtools/dataconverter/nexus_tree.py b/src/pynxtools/dataconverter/nexus_tree.py index 38b78ad5e..1b721cadb 100644 --- a/src/pynxtools/dataconverter/nexus_tree.py +++ b/src/pynxtools/dataconverter/nexus_tree.py @@ -771,15 +771,6 @@ def __repr__(self) -> str: return f"@{self.name} ({self.optionality[:3]})" return f"{self.name} ({self.optionality[:3]})" - # def __repr__(self) -> str: - # inh_str = "\n ".join(str(parent.attrib) for parent in self.inheritance) - # sib_str = "\n ".join(str(sibling) for sibling in self.is_a) - - # inh_part = f"\n inh:\n {inh_str}" if inh_str else "" - # sib_part = f"\n sib: {sib_str}" if sib_str else "" - - # return f"{self.nx_class[2:].upper()}[{self.name.lower()}] ({self.optionality}, nameType: {self.name_type}{inh_part}{sib_part})" - class NexusEntity(NexusNode): """ From 0c6db81d91c64fabc798d4de990f1c21c51d8c79 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Thu, 20 Mar 2025 09:27:52 +0100 Subject: [PATCH 45/52] update defs --- src/pynxtools/definitions | 2 +- src/pynxtools/nexus-version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pynxtools/definitions b/src/pynxtools/definitions index 1790395b5..5dfb75c2c 160000 --- a/src/pynxtools/definitions +++ b/src/pynxtools/definitions @@ -1 +1 @@ -Subproject commit 1790395b5b8e305ecc4c53bd5849dff85c7d7649 +Subproject commit 5dfb75c2ce4227354cf5f83176ecc42f7b6a7e15 diff --git a/src/pynxtools/nexus-version.txt b/src/pynxtools/nexus-version.txt index 517561cb8..2a2e40a38 100644 --- a/src/pynxtools/nexus-version.txt +++ b/src/pynxtools/nexus-version.txt @@ -1 +1 @@ -v2024.02-1889-g1790395b \ No newline at end of file +v2024.02-1894-g5dfb75c2 \ No newline at end of file From 9ac4396e8f3e418077a395c12e6f5d7933477a1d Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Thu, 20 Mar 2025 12:29:58 +0100 Subject: [PATCH 46/52] use uppercase HDF5_Version in NXroot writing --- src/pynxtools/dataconverter/helpers.py | 3 ++- src/pynxtools/testing/nexus_conversion.py | 2 +- tests/dataconverter/test_helpers.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pynxtools/dataconverter/helpers.py b/src/pynxtools/dataconverter/helpers.py index fbcfff02d..d81503083 100644 --- a/src/pynxtools/dataconverter/helpers.py +++ b/src/pynxtools/dataconverter/helpers.py @@ -707,6 +707,7 @@ def is_valid_data_field( # Check enumeration if nxdl_enum is not None and value not in nxdl_enum: + print(value, nxdl_enum, value == nxdl_enum) if nxdl_enum_open: collector.collect_and_log( path, @@ -898,7 +899,7 @@ def update_and_warn(key: str, value: str): f"blob/{get_nexus_version_hash()}", ) update_and_warn("/@NeXus_release", get_nexus_version()) - update_and_warn("/@HDF5_version", ".".join(map(str, h5py.h5.get_libversion()))) + update_and_warn("/@HDF5_Version", ".".join(map(str, h5py.h5.get_libversion()))) update_and_warn("/@h5py_version", h5py.__version__) diff --git a/src/pynxtools/testing/nexus_conversion.py b/src/pynxtools/testing/nexus_conversion.py index 01ccbd849..2f8e99241 100644 --- a/src/pynxtools/testing/nexus_conversion.py +++ b/src/pynxtools/testing/nexus_conversion.py @@ -186,7 +186,7 @@ def check_reproducibility_of_nexus(self, **kwargs): ] IGNORE_SECTIONS: Dict[str, List[str]] = { **reader_ignore_sections, - "ATTRS (//@HDF5_version)": ["DEBUG - value:"], + "ATTRS (//@HDF5_Version)": ["DEBUG - value:"], "ATTRS (//@file_name)": ["DEBUG - value:"], "ATTRS (//@file_time)": ["DEBUG - value:"], "ATTRS (//@file_update_time)": ["DEBUG - value:"], diff --git a/tests/dataconverter/test_helpers.py b/tests/dataconverter/test_helpers.py index 3ee371d3d..883684012 100644 --- a/tests/dataconverter/test_helpers.py +++ b/tests/dataconverter/test_helpers.py @@ -176,7 +176,7 @@ def test_writing_of_root_attributes(caplog): assert "/@file_update_time" in keys_added assert "/@NeXus_repository" in keys_added assert "/@NeXus_release" in keys_added - assert "/@HDF5_version" in keys_added + assert "/@HDF5_Version" in keys_added assert "/@h5py_version" in keys_added assert "/ENTRY[entry]/definition" in keys_added assert "/ENTRY[entry]/definition/@version" in keys_added From a2b3b34b81a480a070f6d4a6c5cb4301cc4b1e52 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Thu, 20 Mar 2025 13:06:07 +0100 Subject: [PATCH 47/52] use main branches for raman and ellips again --- .github/workflows/plugin_test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/plugin_test.yaml b/.github/workflows/plugin_test.yaml index 37b591624..c3780ccd0 100644 --- a/.github/workflows/plugin_test.yaml +++ b/.github/workflows/plugin_test.yaml @@ -24,7 +24,7 @@ jobs: branch: main tests_to_run: tests/. - plugin: pynxtools-ellips - branch: update + branch: main tests_to_run: tests/. - plugin: pynxtools-em branch: main @@ -36,7 +36,7 @@ jobs: branch: update-definitions tests_to_run: tests/. - plugin: pynxtools-raman - branch: update + branch: main tests_to_run: tests/. - plugin: pynxtools-spm branch: FixReaderAlngAppDef From 5ba7b2fb8acec9a4433f59c4117029e6dfdcc9f9 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Thu, 20 Mar 2025 16:20:28 +0100 Subject: [PATCH 48/52] dont collect illegal unit in NXcollection --- src/pynxtools/dataconverter/helpers.py | 1 - src/pynxtools/dataconverter/validation.py | 16 ++++++++++++++ tests/dataconverter/test_validation.py | 26 ++++++++++++++++------- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/src/pynxtools/dataconverter/helpers.py b/src/pynxtools/dataconverter/helpers.py index d81503083..ee3647414 100644 --- a/src/pynxtools/dataconverter/helpers.py +++ b/src/pynxtools/dataconverter/helpers.py @@ -707,7 +707,6 @@ def is_valid_data_field( # Check enumeration if nxdl_enum is not None and value not in nxdl_enum: - print(value, nxdl_enum, value == nxdl_enum) if nxdl_enum_open: collector.collect_and_log( path, diff --git a/src/pynxtools/dataconverter/validation.py b/src/pynxtools/dataconverter/validation.py index 3e84f93fb..f923f11bf 100644 --- a/src/pynxtools/dataconverter/validation.py +++ b/src/pynxtools/dataconverter/validation.py @@ -845,6 +845,22 @@ def startswith_with_variations( else: # check that parent has units node = add_best_matches_for(not_visited_key.rsplit("/", 1)[0], tree) + + # Search if unit is somewhere in an NXcollection + if node is None: + key_path = not_visited_key.replace("@", "") + while "/" in key_path: + key_path = key_path.rsplit("/", 1)[0] # Remove last segment + parent_node = add_best_matches_for(key_path, tree) + if ( + parent_node + and parent_node.type == "group" + and parent_node.nx_class == "NXcollection" + ): + # NXcollection found → break while, continue outer loop + break + continue + if node is None or node.type != "field" or node.unit is None: if not ignore_undocumented: collector.collect_and_log( diff --git a/tests/dataconverter/test_validation.py b/tests/dataconverter/test_validation.py index 39e0c2d6c..80cbbb9b1 100644 --- a/tests/dataconverter/test_validation.py +++ b/tests/dataconverter/test_validation.py @@ -1112,12 +1112,16 @@ def listify_template(data_dict: Template): pytest.param( alter_dict( alter_dict( - TEMPLATE, - "/ENTRY[my_entry]/COLLECTION[collection]/some_field", + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/COLLECTION[collection]/some_field", + 0.5, + ), + "/ENTRY[my_entry]/COLLECTION[collection]/DATA[data]/some_field", 0.5, ), - "/ENTRY[my_entry]/COLLECTION[collection]/DATA[data]/some_field", - 0.5, + "/ENTRY[my_entry]/COLLECTION[collection]/DATA[data]/some_field/@units", + "mm", ), [], id="variadic-nxcollection", @@ -1125,12 +1129,16 @@ def listify_template(data_dict: Template): pytest.param( alter_dict( alter_dict( - TEMPLATE, - "/ENTRY[my_entry]/named_collection/some_field", + alter_dict( + TEMPLATE, + "/ENTRY[my_entry]/named_collection/some_field", + 0.5, + ), + "/ENTRY[my_entry]/named_collection/DATA[data]/some_field", 0.5, ), - "/ENTRY[my_entry]/named_collection/DATA[data]/some_field", - 0.5, + "/ENTRY[my_entry]/named_collection/DATA[data]/some_field/@units", + "mm", ), [], id="nonvariadic-nxcollection", @@ -1153,6 +1161,8 @@ def format_error_message(msg: str) -> str: "baseclass-field-with-illegal-unit", "open-enum-with-new-item", "baseclass-open-enum-with-new-item", + "variadic-nxcollection", + "nonvariadic-nxcollection", ): with caplog.at_level(logging.INFO): assert validate_dict_against("NXtest", data_dict)[0] From 737c5e311683beaa66b571796f71b4eb90e55139 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Thu, 20 Mar 2025 17:36:10 +0100 Subject: [PATCH 49/52] fix unit examples in NOMAD --- src/pynxtools/nomad/schema.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pynxtools/nomad/schema.py b/src/pynxtools/nomad/schema.py index c86781401..f9fc1c4ec 100644 --- a/src/pynxtools/nomad/schema.py +++ b/src/pynxtools/nomad/schema.py @@ -774,7 +774,10 @@ def __create_field(xml_node: ET.Element, container: Section) -> Quantity: from nomad.units import ureg quantity = 1 * ureg(nx_dimensionality) - dimensionality = quantity.dimensionality + if quantity.dimensionality == "dimensionless": + dimensionality = "1" + else: + dimensionality = str(quantity.dimensionality) except ( pint.errors.UndefinedUnitError, pint.errors.DefinitionSyntaxError, From 7b590ccc3cab2309daff0113985769613ae0e03a Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Thu, 20 Mar 2025 17:36:26 +0100 Subject: [PATCH 50/52] update definitions once more --- src/pynxtools/definitions | 2 +- src/pynxtools/nexus-version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pynxtools/definitions b/src/pynxtools/definitions index 5dfb75c2c..a85e10cd0 160000 --- a/src/pynxtools/definitions +++ b/src/pynxtools/definitions @@ -1 +1 @@ -Subproject commit 5dfb75c2ce4227354cf5f83176ecc42f7b6a7e15 +Subproject commit a85e10cd0289f4e44b0fec011ff54703e6705383 diff --git a/src/pynxtools/nexus-version.txt b/src/pynxtools/nexus-version.txt index 2a2e40a38..c137d59d6 100644 --- a/src/pynxtools/nexus-version.txt +++ b/src/pynxtools/nexus-version.txt @@ -1 +1 @@ -v2024.02-1894-g5dfb75c2 \ No newline at end of file +v2024.02-1913-ga85e10cd \ No newline at end of file From b9fbc9d9d2044236164d836a06feb3c71f664301 Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Thu, 20 Mar 2025 17:36:54 +0100 Subject: [PATCH 51/52] update CITATION.cff before release --- CITATION.cff | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CITATION.cff b/CITATION.cff index 67d45ad01..cf46d9c00 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -4,7 +4,7 @@ message: If you use this software, please cite it using the metadata from this file. type: software -version: 0.9.3 +version: 0.10.0 authors: - given-names: Sherjeel family-names: Shabih From b4549168e0cf32716befe3447acfde308a29ad4b Mon Sep 17 00:00:00 2001 From: Lukas Pielsticker <50139597+lukaspie@users.noreply.github.com> Date: Thu, 20 Mar 2025 17:45:34 +0100 Subject: [PATCH 52/52] reset plugin test branches --- .github/workflows/plugin_test.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/plugin_test.yaml b/.github/workflows/plugin_test.yaml index c3780ccd0..32fd9bbfd 100644 --- a/.github/workflows/plugin_test.yaml +++ b/.github/workflows/plugin_test.yaml @@ -30,19 +30,19 @@ jobs: branch: main tests_to_run: tests/. - plugin: pynxtools-igor - branch: update-definitions + branch: main tests_to_run: tests/. - plugin: pynxtools-mpes - branch: update-definitions + branch: main tests_to_run: tests/. - plugin: pynxtools-raman branch: main tests_to_run: tests/. - plugin: pynxtools-spm - branch: FixReaderAlngAppDef + branch: main tests_to_run: tests/. - plugin: pynxtools-xps - branch: update-to-new-nxmpes + branch: main tests_to_run: tests/. - plugin: pynxtools-xrd branch: main