read metadata and cards from tar without extracting multi-dim arrays

src/eko/io/metadata.py

@@ -69,15 +69,27 @@ def load(cls, path: os.PathLike):
         # patch if necessary
         if data_version == 1:
             if version.major == 0 and version.minor == 13:
-                raw = v1.update_metadata(paths, raw)
+                raw = v1.update_metadata(raw)
             elif version.major == 0 and version.minor == 14:
-                raw = v2.update_metadata(paths, raw)
+                raw = v2.update_metadata(raw)
 
         # now we are ready
         content = cls.from_dict(raw)
         content._path = path
         return content
 
+    @classmethod
+    def from_raw(cls, raw: dict) -> "Metadata":
+        """Build metadata from raw yaml, applying legacy patches."""
+        version = parse(raw["version"])
+        data_version = int(raw["data_version"])
+        if data_version == 1:
+            if version.major == 0 and version.minor == 13:
+                raw = v1.update_metadata(raw)
+            elif version.major == 0 and version.minor == 14:
+                raw = v2.update_metadata(raw)
+        return cls.from_dict(raw)
+
     def update(self):
         """Update the disk copy of metadata."""
         if self._path is None:

src/eko/io/runcards.py

@@ -4,12 +4,15 @@
 squared value, for consistency. Squares are consistently taken inside.
 """
 
+import pathlib
+import tarfile
 from dataclasses import dataclass
 from math import nan
 from typing import List, Optional, Union
 
 import numpy as np
 import numpy.typing as npt
+import yaml
 
 from .. import basis_rotation as br
 from .. import interpolation, msbar_masses
@@ -19,7 +22,10 @@
 from ..quantities import heavy_quarks as hq
 from ..quantities.couplings import CouplingsInfo
 from ..quantities.heavy_quarks import HeavyInfo, QuarkMassScheme
+from . import v1, v2
 from .dictlike import DictLike
+from .metadata import Metadata
+from .paths import METADATAFILE, OPERATORFILE, THEORYFILE
 from .types import (
     EvolutionMethod,
     InversionMethod,
@@ -78,6 +84,15 @@ def __post_init__(self):
         if self.matching_order is None:
             self.matching_order = (self.order[0] - 1, 0)
 
+    @classmethod
+    def from_raw(cls, raw: dict, data_version: int) -> "TheoryCard":
+        """Build a theory card from a raw dictionary."""
+        if data_version == 1:
+            raw = v1.update_theory(raw)
+        if data_version == 2:
+            raw = v2.update_theory(raw)
+        return cls.from_dict(raw)
+
 
 @dataclass
 class Debug(DictLike):
@@ -170,6 +185,15 @@ def pids(self):
         """Internal flavor basis, used for computation."""
         return np.array(br.flavor_basis_pids)
 
+    @classmethod
+    def from_raw(cls, raw: dict, data_version: int, theory_raw) -> "OperatorCard":
+        """Build an operator card from a raw dictionary."""
+        if data_version == 1:
+            raw = v1.update_operator(raw, theory_raw)
+        if data_version == 2:
+            raw = v2.update_operator(raw, theory_raw)
+        return cls.from_dict(raw)
+
 
 Card = Union[TheoryCard, OperatorCard]
 
@@ -343,3 +367,37 @@ def masses(theory: TheoryCard, evmeth: EvolutionMethod) -> List[SquaredScale]:
         return [mq.value**2 for mq in theory.heavy.masses]
 
     raise ValueError(f"Unknown mass scheme '{theory.heavy.masses_scheme}'")
+
+
+def _read_yaml_members(tar, filenames):
+    """Extract and parse several yaml files in a single pass."""
+    wanted = set(filenames)
+    found = {}
+    for member in tar.getmembers():
+        name = pathlib.Path(member.name).name
+        if name in wanted:
+            extracted = tar.extractfile(member)
+            if extracted is not None:
+                found[name] = yaml.safe_load(extracted.read())
+    missing = wanted - found.keys()
+    if missing:
+        raise KeyError(f"{missing} not found in {tar.name}")
+    return found
+
+
+def read_eko_cards(eko_path):
+    """Read metadata, theory and operator cards from an EKO archive.
+
+    The (large) operators are never extracted, only the small yaml files.
+    """
+    with tarfile.open(eko_path) as tar:
+        raw_files = _read_yaml_members(tar, [METADATAFILE, THEORYFILE, OPERATORFILE])
+
+    raw_meta = raw_files[METADATAFILE]
+    raw_theory = raw_files[THEORYFILE]
+    raw_operator = raw_files[OPERATORFILE]
+    metadata = Metadata.from_raw(raw_meta)
+    data_version = metadata.data_version
+    operator = OperatorCard.from_raw(raw_operator, data_version, raw_theory)
+    theory = TheoryCard.from_raw(raw_theory, data_version)
+    return metadata, theory, operator

src/eko/io/struct.py

@@ -14,7 +14,7 @@
 import yaml
 
 from .. import interpolation
-from . import exceptions, raw, v1, v2
+from . import exceptions, raw
 from .access import AccessConfigs
 from .inventory import Inventory
 from .items import Evolution, Matching, Operator, Recipe, Target
@@ -133,25 +133,14 @@ def evolgrid(self) -> List[EPoint]:
     def theory_card(self):
         """Provide theory card, retrieving from the dump."""
         raw_th = yaml.safe_load(self.paths.theory_card.read_text(encoding="utf-8"))
-        if self.metadata.data_version in [1]:
-            raw_th = v1.update_theory(raw_th)
-        if self.metadata.data_version in [2]:
-            raw_th = v2.update_theory(raw_th)
-        return TheoryCard.from_dict(raw_th)
+        return TheoryCard.from_raw(raw_th, self.metadata.data_version)
 
     @property
     def operator_card(self):
         """Provide operator card, retrieving from the dump."""
         raw_op = yaml.safe_load(self.paths.operator_card.read_text(encoding="utf-8"))
-        if self.metadata.data_version in [1]:
-            # here we need to read also the theory card
-            raw_th = yaml.safe_load(self.paths.theory_card.read_text(encoding="utf-8"))
-            raw_op = v1.update_operator(raw_op, raw_th)
-        if self.metadata.data_version in [2]:
-            # here we need to read also the theory card
-            raw_th = yaml.safe_load(self.paths.theory_card.read_text(encoding="utf-8"))
-            raw_op = v2.update_operator(raw_op, raw_th)
-        return OperatorCard.from_dict(raw_op)
+        raw_th = yaml.safe_load(self.paths.theory_card.read_text(encoding="utf-8"))
+        return OperatorCard.from_raw(raw_op, self.metadata.data_version, raw_th)
 
     # persistency control
     # -------------------

src/eko/io/v1.py

@@ -4,10 +4,8 @@
 that API version.
 """
 
-from .paths import InternalPaths
 
-
-def update_metadata(paths: InternalPaths, raw: dict) -> dict:
+def update_metadata(raw: dict) -> dict:
     """Modify the raw metadata to the new format.
 
     Parameters

src/eko/io/v2.py

@@ -4,10 +4,8 @@
 exactly that API version.
 """
 
-from .paths import InternalPaths
 
-
-def update_metadata(paths: InternalPaths, raw: dict) -> dict:
+def update_metadata(raw: dict) -> dict:
     """Modify the raw metadata to the new format.
 
     Parameters

tests/conftest.py

@@ -68,6 +68,11 @@ def operator_card():
     return card
 
 
+@pytest.fixture
+def legacy_eko_filenames():
+    return ["v1-0.13.tar", "v1-0.14.tar", "v3.tar"]
+
+
 class EKOFactory:
     def __init__(self, theory: TheoryCard, operator: OperatorCard, path: os.PathLike):
         self.path = path

tests/eko/io/test_legacy.py

@@ -18,8 +18,8 @@
 EP = (10000.0, 4)
 
 
-def test_read_legacy():
-    for name in ["v1-0.13.tar", "v1-0.14.tar", "v3.tar"]:
+def test_read_legacy(legacy_eko_filenames):
+    for name in legacy_eko_filenames:
         with eko.EKO.read(TEST_DATA_DIR / name) as evolution_operator:
             # Check the cards
             assert isinstance(

tests/eko/io/test_struct.py

@@ -10,6 +10,8 @@
 from eko import EKO, interpolation
 from eko.io import struct
 from eko.io.items import Target
+from eko.io.metadata import Metadata
+from eko.io.runcards import OperatorCard, TheoryCard, read_eko_cards
 from tests.conftest import EKOFactory
 
 
@@ -197,6 +199,18 @@ def test_load_opened(self, tmp_path: pathlib.Path, eko_factory: EKOFactory):
 
         assert read_closed.metadata == read_opened.metadata
 
+    def test_read_eko_cards(self, legacy_eko_filenames):
+        """Load metadata and both YAML cards from legacy EKO archives."""
+        data_dir = pathlib.Path(__file__).parents[2] / "data"
+        for filename in legacy_eko_filenames:
+            metadata, theory, operator = read_eko_cards(
+                data_dir / filename
+            )
+
+            assert isinstance(metadata, Metadata)
+            assert isinstance(theory, TheoryCard)
+            assert isinstance(operator, OperatorCard)
+
     def test_version(self, tmp_path: pathlib.Path, eko_factory: EKOFactory):
         """Test asserted version. Should either be supported version, or have a postrelease addition"""
         eko = eko_factory.get()