diff --git a/src/modules/client/actioncache.py b/src/modules/client/actioncache.py new file mode 100644 index 000000000..89ccbcb6d --- /dev/null +++ b/src/modules/client/actioncache.py @@ -0,0 +1,686 @@ +#!/usr/bin/python3 +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright 2026 OmniOS Community Edition (OmniOSce) Association. +# + +"""The installed-action cache. + +This module maintains a per-image sqlite3 database recording, for every +installed package, the stripped form of each globally-identical action +that the package delivers, keyed by action name and key attribute value, +together with a table of the key attribute values for which the installed +actions conflict with one another. + +""" + +import errno +import os +import sqlite3 +from urllib.parse import quote + +import pkg.actions +import pkg.client.imageplan as imageplan +import pkg.client.progress as progress +import pkg.misc as misc +import pkg.portable as portable + +CACHE_BASENAME = "actions.sqlite" + +SCHEMA_VERSION = 1 + +# The number of host variables used per statement when binding a large +# set of values. sqlite guarantees at least 999. +_CHUNK = 500 + +# Batch size for action row inserts. +_INSERT_BATCH = 5000 + +_SCHEMA = [ + """CREATE TABLE meta ( + name TEXT PRIMARY KEY, + value TEXT NOT NULL + ) WITHOUT ROWID""", + """CREATE TABLE packages ( + pkg_id INTEGER PRIMARY KEY, + fmri TEXT NOT NULL UNIQUE + )""", + """CREATE TABLE actions ( + pkg_id INTEGER NOT NULL, + aname TEXT NOT NULL, + keyval TEXT NOT NULL, + act TEXT NOT NULL + )""", + """CREATE TABLE conflicts ( + ns TEXT NOT NULL, + keyval TEXT NOT NULL, + PRIMARY KEY (ns, keyval) + ) WITHOUT ROWID""", +] + +_INDICES = [ + "CREATE INDEX actions_ix_key ON actions (keyval, aname)", + "CREATE INDEX actions_ix_pkg ON actions (pkg_id)", +] + + +class ActionCacheError(Exception): + """Base exception for installed-action cache errors.""" + + +class ReadOnlyCacheError(ActionCacheError): + """Raised when the database cannot be opened for writing.""" + + +def _conflict_groups(): + """Return a dictionary mapping each namespace group that contains + at least one globally-identical action type to the set of + globally-identical action names within that group. + + Namespace group values themselves are not stable across releases + (see pkg.actions.generic.NSG), which is why they are always derived + at runtime and never stored in the database.""" + + groups = {} + for name, klass in pkg.actions.types.items(): + if not klass.globally_identical: + continue + groups.setdefault(klass.namespace_group, set()).add(name) + return groups + + +def _excludes_signature(image): + """Return a canonical string describing the variants and facets + currently configured for 'image'. When this changes, the set of + actions admitted by the image's excludes may change for any + installed package, so the cache must be rebuilt in full.""" + + return str( + ( + sorted((str(k), str(v)) for k, v in image.cfg.variants.items()), + sorted((str(k), str(v)) for k, v in image.cfg.facets.items()), + ) + ) + + +class ActionCache(object): + """Manages the installed-action cache database for an image. + + All write operations assume the caller holds the image lock; the + only concurrency handled here is unprivileged readers observing a + database replaced by rename, which is safe because an open file + descriptor keeps the old copy alive.""" + + def __init__(self, image, cache_dir): + self.__image = image + self.__dir = cache_dir + self.__path = os.path.join(cache_dir, CACHE_BASENAME) + self.__con = None + self.__rw = False + self.__conid = None + + @property + def cache_dir(self): + return self.__dir + + @property + def pathname(self): + return self.__path + + def close(self): + if self.__con: + try: + self.__con.close() + except sqlite3.Error: + pass + self.__con = None + self.__rw = False + self.__conid = None + + def __fileid(self): + """Return an identity token for the file currently at the + database path, or None if it does not exist.""" + + try: + st = os.stat(self.__path) + return (st.st_dev, st.st_ino) + except EnvironmentError: + return None + + def __check_replaced(self): + """Drop the cached connection if the file at the database + path is no longer the file the connection was opened on (a + full rebuild replaces the database by rename).""" + + if self.__con and self.__conid != self.__fileid(): + self.close() + + def __connect(self, mode): + con = sqlite3.connect( + "file:{0}?mode={1}".format(quote(self.__path), mode), + uri=True, + check_same_thread=False, + isolation_level=None, + ) + con.execute("PRAGMA temp_store = MEMORY") + return con + + def __open_ro(self): + """Return a read-only connection to the database, or None if + it does not exist or cannot be opened.""" + + self.__check_replaced() + if self.__con: + return self.__con + try: + self.__con = self.__connect("ro") + except sqlite3.OperationalError: + return None + self.__rw = False + self.__conid = self.__fileid() + return self.__con + + def __open_rw(self): + """Return a read-write connection to the database, raising + ReadOnlyCacheError if it exists but cannot be written.""" + + self.__check_replaced() + if self.__con and self.__rw: + return self.__con + self.close() + # Verify writability of both the database file and its + # directory (needed for the rollback journal) up front, so + # that callers see a single exception type for the fallback + # path; sqlite otherwise reports failures only at the first + # actual write. + try: + fd = os.open(self.__path, os.O_WRONLY) + os.close(fd) + probe = self.__path + ".tmp" + fd = os.open( + probe, + os.O_CREAT | os.O_WRONLY, + misc.PKG_FILE_MODE, + ) + os.close(fd) + os.unlink(probe) + except EnvironmentError as e: + if e.errno in (errno.EACCES, errno.EROFS, errno.EPERM): + raise ReadOnlyCacheError(str(e)) + raise + try: + con = self.__connect("rw") + except sqlite3.OperationalError as e: + raise ReadOnlyCacheError(str(e)) + con.execute("PRAGMA synchronous = NORMAL") + self.__con = con + self.__rw = True + self.__conid = self.__fileid() + return con + + def __installed_pfmris(self): + """Return a dictionary mapping installed package fmri strings + to their PkgFmri objects according to the image's installed + catalog.""" + + return dict( + (str(pfmri), pfmri) for pfmri in self.__image.gen_installed_pkgs() + ) + + def __usable(self, con): + """Return True if the database was completely built by a + compatible version of this code for the image's current + variants and facets.""" + + try: + uv = con.execute("PRAGMA user_version").fetchone()[0] + if uv != SCHEMA_VERSION: + return False + meta = dict(con.execute("SELECT name, value FROM meta")) + except sqlite3.DatabaseError: + return False + if meta.get("complete") != "1": + return False + return meta.get("excludes") == _excludes_signature(self.__image) + + def is_fresh(self): + """Return True if the database exists, is usable, and is + consistent with the image's installed catalog.""" + + con = self.__open_ro() + if con is None: + return False + try: + if not self.__usable(con): + return False + dbf = set(r[0] for r in con.execute("SELECT fmri FROM packages")) + except sqlite3.DatabaseError: + return False + return dbf == set(self.__installed_pfmris()) + + def update(self, progtrack=None): + """Bring the database into line with the installed catalog, + incrementally where possible, rebuilding otherwise. Raises + ReadOnlyCacheError or EnvironmentError (EACCES/EROFS) when the + cache is not writable.""" + + if not progtrack: + progtrack = progress.NullProgressTracker() + + if not os.path.exists(self.__path): + self.rebuild(progtrack=progtrack) + return + + try: + con = self.__open_rw() + if not self.__usable(con): + self.rebuild(progtrack=progtrack) + return + dbf = dict(con.execute("SELECT fmri, pkg_id FROM packages")) + except sqlite3.DatabaseError: + # Corrupt database; replace it. A cache that cannot be + # written raises ReadOnlyCacheError instead, which is not + # handled here. + self.rebuild(progtrack=progtrack) + return + + inst = self.__installed_pfmris() + extra = [pkg_id for f, pkg_id in dbf.items() if f not in inst] + missing = [f for f in inst if f not in dbf] + if not extra and not missing: + return + + progtrack.job_start(progtrack.JOB_FAST_LOOKUP) + try: + # (aname -> set(keyval)) touched by this update; conflict + # state is recomputed for these keys. + affected = {} + cur = con.cursor() + cur.execute("BEGIN IMMEDIATE") + for i in range(0, len(extra), _CHUNK): + chunk = extra[i : i + _CHUNK] + qs = ",".join("?" * len(chunk)) + for aname, keyval in cur.execute( + "SELECT DISTINCT aname, keyval FROM actions" + " WHERE pkg_id IN ({0})".format(qs), + chunk, + ).fetchall(): + affected.setdefault(aname, set()).add(keyval) + cur.execute( + "DELETE FROM actions WHERE pkg_id IN ({0})".format(qs), + chunk, + ) + cur.execute( + "DELETE FROM packages WHERE pkg_id IN ({0})".format(qs), + chunk, + ) + progtrack.job_add_progress(progtrack.JOB_FAST_LOOKUP) + + for f in missing: + progtrack.job_add_progress(progtrack.JOB_FAST_LOOKUP) + cur.execute("INSERT INTO packages (fmri) VALUES (?)", (f,)) + pkg_id = cur.lastrowid + batch = [] + for _f, aname, keyval, act in self.__gen_package_rows(inst[f]): + affected.setdefault(aname, set()).add(keyval) + batch.append((pkg_id, aname, keyval, act)) + cur.executemany("INSERT INTO actions VALUES (?,?,?,?)", batch) + + self.__refresh_conflicts(con, progtrack, affected) + cur.execute("COMMIT") + except BaseException: + con.execute("ROLLBACK") + raise + finally: + progtrack.job_done(progtrack.JOB_FAST_LOOKUP) + + def __gen_package_rows(self, pfmri): + """Yield (fmri string, action name, key attribute value, + stripped action string) for every globally-identical action + delivered by 'pfmri' under the image's current excludes.""" + + excludes = self.__image.list_excludes() + m = self.__image.get_manifest(pfmri, ignore_excludes=True) + f = str(pfmri) + for act in m.gen_actions(excludes=excludes): + if not act.globally_identical: + continue + act.strip() + yield f, act.name, act.attrs[act.key_attr], str(act) + + def rebuild(self, progtrack=None): + """Rebuild the database from scratch from the manifests of the + installed packages, atomically replacing any existing + database. Raises EnvironmentError with EACCES/EROFS when the + cache directory is not writable.""" + + if not progtrack: + progtrack = progress.NullProgressTracker() + + self.close() + + if not os.path.exists(self.__dir): + os.makedirs(self.__dir) + + tmp_path = self.__path + ".tmp" + # Probe writability with a plain open so that permission + # problems surface as EnvironmentError with a meaningful errno + # rather than a generic sqlite error, and remove any leftover + # temporary database from an interrupted build. + fd = os.open( + tmp_path, + os.O_CREAT | os.O_WRONLY | os.O_TRUNC, + misc.PKG_FILE_MODE, + ) + os.close(fd) + + progtrack.job_start(progtrack.JOB_FAST_LOOKUP) + con = sqlite3.connect( + tmp_path, check_same_thread=False, isolation_level=None + ) + try: + # Durability is provided by the rename into place below; a + # partially-written temporary file is never visible. + con.execute("PRAGMA journal_mode = OFF") + con.execute("PRAGMA synchronous = OFF") + con.execute("PRAGMA temp_store = MEMORY") + con.execute("BEGIN") + for ddl in _SCHEMA: + con.execute(ddl) + + cur = con.cursor() + batch = [] + for f, pfmri in self.__installed_pfmris().items(): + progtrack.job_add_progress(progtrack.JOB_FAST_LOOKUP) + cur.execute("INSERT INTO packages (fmri) VALUES (?)", (f,)) + pkg_id = cur.lastrowid + for _f, aname, keyval, act in self.__gen_package_rows(pfmri): + batch.append((pkg_id, aname, keyval, act)) + if len(batch) >= _INSERT_BATCH: + cur.executemany( + "INSERT INTO actions VALUES (?,?,?,?)", batch + ) + batch = [] + if batch: + cur.executemany("INSERT INTO actions VALUES (?,?,?,?)", batch) + + progtrack.job_add_progress(progtrack.JOB_FAST_LOOKUP) + for ddl in _INDICES: + con.execute(ddl) + + self.__refresh_conflicts(con, progtrack, None) + + con.execute( + "INSERT INTO meta VALUES ('excludes', ?)", + (_excludes_signature(self.__image),), + ) + con.execute("INSERT INTO meta VALUES ('complete', '1')") + con.execute("PRAGMA user_version = {0:d}".format(SCHEMA_VERSION)) + con.execute("COMMIT") + con.close() + con = None + os.chmod(tmp_path, misc.PKG_FILE_MODE) + portable.rename(tmp_path, self.__path) + except BaseException: + if con: + con.close() + try: + os.unlink(tmp_path) + except OSError: + pass + raise + finally: + progtrack.job_done(progtrack.JOB_FAST_LOOKUP) + + def __refresh_conflicts(self, con, progtrack, affected): + """Recompute the conflicts table. + + If 'affected' is None the whole table is rebuilt from the + actions table; otherwise it is a dictionary mapping action + names to the sets of key attribute values whose conflict state + must be re-evaluated.""" + + full = affected is None + if full: + con.execute("DELETE FROM conflicts") + + for ns, names in _conflict_groups().items(): + # The namespace group value itself is unstable across + # releases, so conflicts rows are keyed by the smallest + # action name in the group instead. + rep = min(names) + onames = sorted(names) + nq = ",".join("?" * len(onames)) + # Refcountable, globally-identical types (dir, link, + # hardlink, ...) may be delivered by any number of + # packages provided the actions are identical, so groups + # that are homogeneous in both type and content are + # provably conflict-free and are filtered out in SQL + # before any action parsing happens. + refgi = sorted( + n for n in names if pkg.actions.types[n].refcountable + ) + rq = ",".join("?" * len(refgi)) + + if full: + keyvals = None + else: + keyvals = set() + for aname in names: + keyvals |= affected.get(aname, set()) + if not keyvals: + continue + keyvals = sorted(keyvals) + + bad = set() + for kchunk in self.__chunks(keyvals): + progtrack.job_add_progress(progtrack.JOB_FAST_LOOKUP) + where = "aname IN ({0})".format(nq) + params = list(onames) + if kchunk is not None: + where += " AND keyval IN ({0})".format( + ",".join("?" * len(kchunk)) + ) + params += kchunk + cands = [ + r[0] + for r in con.execute( + "SELECT keyval FROM actions" + " WHERE {0} GROUP BY keyval" + " HAVING COUNT(*) > 1 AND NOT" + " (COUNT(DISTINCT aname) = 1" + " AND COUNT(DISTINCT act) = 1" + " AND MIN(aname) IN ({1}))".format(where, rq), + params + refgi, + ) + ] + + for cchunk in self.__chunks(cands, none_ok=False): + groups = {} + for keyval, act, f in con.execute( + "SELECT a.keyval, a.act, p.fmri" + " FROM actions a" + " JOIN packages p ON p.pkg_id = a.pkg_id" + " WHERE a.aname IN ({0})" + " AND a.keyval IN ({1})".format( + nq, ",".join("?" * len(cchunk)) + ), + onames + cchunk, + ): + groups.setdefault(keyval, []).append( + (pkg.actions.fromstr(act), f) + ) + for keyval, actions in groups.items(): + if imageplan.ImagePlan._check_action_group(ns, actions): + bad.add(keyval) + + if kchunk is not None: + con.execute( + "DELETE FROM conflicts WHERE ns = ?" + " AND keyval IN ({0})".format( + ",".join("?" * len(kchunk)) + ), + [rep] + kchunk, + ) + con.executemany( + "INSERT OR REPLACE INTO conflicts VALUES (?, ?)", + [(rep, k) for k in sorted(bad)], + ) + + @staticmethod + def __chunks(vals, none_ok=True): + """Split 'vals' into lists of at most _CHUNK items. If 'vals' + is None and 'none_ok' is set, yield a single None, meaning 'no + restriction'.""" + + if vals is None: + if none_ok: + yield None + return + for i in range(0, len(vals), _CHUNK): + yield vals[i : i + _CHUNK] + + def get_actions(self, anames, keys): + """Yield (key attribute value, fmri string, stripped action + string) tuples for every cached action whose action name is in + 'anames' and whose key attribute value is in 'keys'.""" + + con = self.__open_ro() + if con is None: + raise ActionCacheError( + "installed-action cache disappeared: {0}".format(self.__path) + ) + anames = sorted(anames) + nq = ",".join("?" * len(anames)) + keys = list(keys) + for i in range(0, len(keys), _CHUNK): + chunk = keys[i : i + _CHUNK] + yield from con.execute( + "SELECT a.keyval, p.fmri, a.act" + " FROM actions a" + " JOIN packages p ON p.pkg_id = a.pkg_id" + " WHERE a.keyval IN ({0}) AND a.aname IN ({1})" + " ORDER BY a.keyval, a.aname, p.fmri, a.act".format( + ",".join("?" * len(chunk)), nq + ), + chunk + anames, + ) + + def has_conflicts(self): + """Return True if any installed actions conflict with each + other (or if that cannot be determined).""" + + con = self.__open_ro() + if con is None: + return True + try: + return ( + con.execute( + "SELECT EXISTS (SELECT 1 FROM conflicts)" + ).fetchone()[0] + == 1 + ) + except sqlite3.DatabaseError: + return True + + def conflicting_keys(self): + """Return the set of key attribute values for which installed + actions conflict, or None if the database is unusable.""" + + con = self.__open_ro() + if con is None: + return None + try: + return set( + r[0] for r in con.execute("SELECT keyval FROM conflicts") + ) + except sqlite3.DatabaseError: + return None + + def copy_to(self, cache_dir): + """Return a new ActionCache rooted at 'cache_dir', seeded with + a copy of this cache's database if one exists. Used to give + unprivileged users a reconcilable private copy.""" + + other = ActionCache(self.__image, cache_dir) + try: + misc.copyfile(self.__path, other.pathname) + os.chmod(other.pathname, misc.PKG_FILE_MODE) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + return other + + def __dumpdb(self): + """Return the comparable content of the database as a tuple + of (action rows, conflict rows) frozensets.""" + + con = self.__open_ro() + if con is None: + raise ActionCacheError( + "installed-action cache disappeared: {0}".format(self.__path) + ) + return ( + frozenset( + con.execute( + "SELECT p.fmri, a.aname, a.keyval, a.act" + " FROM actions a" + " JOIN packages p ON p.pkg_id = a.pkg_id" + ) + ), + frozenset(con.execute("SELECT ns, keyval FROM conflicts")), + ) + + def selfcheck(self): + """Compare the database against one rebuilt from scratch from + the installed manifests, returning None if they are identical + or a string describing the differences. This is the backend + for the -D actioncache-verify=1 debug feature, used to + validate incremental maintenance.""" + + ref = ActionCache(self.__image, self.__image.temporary_dir()) + try: + ref.rebuild() + rows, conf = self.__dumpdb() + rrows, rconf = ref.__dumpdb() + finally: + ref.close() + try: + os.unlink(ref.pathname) + except OSError: + pass + + if rows == rrows and conf == rconf: + return None + + def describe(name, mine, theirs): + missing = theirs - mine + extra = mine - theirs + out = [] + if missing: + out.append( + "{0:d} missing {1} row(s), e.g. {2}".format( + len(missing), name, sorted(missing)[0] + ) + ) + if extra: + out.append( + "{0:d} unexpected {1} row(s), e.g. {2}".format( + len(extra), name, sorted(extra)[0] + ) + ) + return out + + return "; ".join( + describe("action", rows, rrows) + describe("conflict", conf, rconf) + ) diff --git a/src/modules/client/image.py b/src/modules/client/image.py index 627060fe5..163d3add3 100644 --- a/src/modules/client/image.py +++ b/src/modules/client/image.py @@ -45,6 +45,7 @@ import pkg.actions import pkg.catalog +import pkg.client.actioncache as actioncache import pkg.client.api_errors as apx import pkg.client.bootenv as bootenv import pkg.client.history as history @@ -228,9 +229,9 @@ def __init__( # dependency but removed because obsolete self.__group_obsolete = None - # The action dictionary that's returned by __load_actdict. - self.__actdict = None - self.__actdict_timestamp = None + # Memoized installed-action cache handle; see + # get_action_cache(). + self.__actioncache = None self.__property_overrides = {"property": props} @@ -3771,193 +3772,83 @@ def gen_tracked_stems(self): if a.name == "depend" and a.attrs["type"] == "group": yield (f, self.strtofmri(a.attrs["fmri"]).pkg_name) - def _create_fast_lookups(self, progtrack=None): - """Create an on-disk database mapping action name and key - attribute value to the action string comprising the unique - attributes of the action, for all installed actions. This is - done with a file mapping the tuple to an offset into a second - file, where those actions are kept. Once the offsets are loaded - into memory, it is simple to seek into the second file to the - given offset and read until you hit an action that doesn't - match.""" + def get_action_cache(self, progtrack=None): + """Return an ActionCache open for reading and consistent with + the installed package catalog, creating or updating the + on-disk database as necessary. Unprivileged users receive a + private, temporary copy when the image's own cache is missing + or out of date.""" if not progtrack: progtrack = progress.NullProgressTracker() - self.__actdict = None - self.__actdict_timestamp = None - stripped_path = os.path.join( - self.__action_cache_dir, "actions.stripped" - ) - offsets_path = os.path.join(self.__action_cache_dir, "actions.offsets") - conflicting_keys_path = os.path.join( - self.__action_cache_dir, "keys.conflicting" - ) + cache = self.__actioncache + if cache is None: + cache = actioncache.ActionCache(self, self.__action_cache_dir) - excludes = self.list_excludes() - heap = [] + if cache.is_fresh(): + self.__actioncache = cache + return cache - # nsd is the "name-space dictionary." It maps action name - # spaces (see action.generic for more information) to - # dictionaries which map keys to pairs which contain an action - # with that key and the pfmri of the package which delivered the - # action. - nsd = {} - - from heapq import heappush, heappop - - progtrack.job_start(progtrack.JOB_FAST_LOOKUP) + try: + cache.update(progtrack=progtrack) + self.__actioncache = cache + return cache + except actioncache.ReadOnlyCacheError: + pass + except EnvironmentError as e: + if e.errno not in (errno.EACCES, errno.EROFS): + raise - for pfmri in self.gen_installed_pkgs(): - progtrack.job_add_progress(progtrack.JOB_FAST_LOOKUP) - m = self.get_manifest(pfmri, ignore_excludes=True) - for act in m.gen_actions(excludes=excludes): - if not act.globally_identical: - continue - act.strip() - heappush(heap, (act.name, act.attrs[act.key_attr], pfmri, act)) - nsd.setdefault(act.namespace_group, {}) - nsd[act.namespace_group].setdefault(act.attrs[act.key_attr], []) - nsd[act.namespace_group][act.attrs[act.key_attr]].append( - (act, pfmri) - ) + # The image's cache isn't writable; reconcile a private copy + # in a temporary directory instead. + cache.close() + cache = cache.copy_to(self.temporary_dir()) + cache.update(progtrack=progtrack) + self.__actioncache = cache + return cache - progtrack.job_add_progress(progtrack.JOB_FAST_LOOKUP) + def _create_fast_lookups(self, progtrack=None): + """Rebuild the installed-action cache database from scratch. + Most callers should use get_action_cache() instead, which + reconciles an existing database incrementally.""" - # If we can't write the temporary files, then there's no point - # in producing actdict because it depends on a synchronized - # stripped actions file. + self.__actioncache = None + cache = actioncache.ActionCache(self, self.__action_cache_dir) try: - actdict = {} - sf, sp = self.temporary_file(close=False) - of, op = self.temporary_file(close=False) - bf, bp = self.temporary_file(close=False) - - sf = os.fdopen(sf, "w") - of = os.fdopen(of, "w") - bf = os.fdopen(bf, "w") - - # We need to make sure the files are coordinated. - timestamp = int(time.time()) - sf.write("VERSION 1\n{0}\n".format(timestamp)) - of.write("VERSION 2\n{0}\n".format(timestamp)) - # The conflicting keys file doesn't need a timestamp - # because it's not coordinated with the stripped or - # offsets files and the result of loading it isn't - # reused by this class. - bf.write("VERSION 1\n") - - cnt, offset_update_bytes = 0, 0 - last_name, last_key, last_offset = None, None, sf.tell() - while heap: - # This is a tight loop, so try to avoid burning - # CPU calling into the progress tracker - # excessively. - if len(heap) % 100 == 0: - progtrack.job_add_progress(progtrack.JOB_FAST_LOOKUP) - item = heappop(heap) - fmri, act = item[2:] - key = act.attrs[act.key_attr] - if act.name != last_name or key != last_key: - if last_name is None: - assert last_key is None - cnt += 1 - last_name = act.name - last_key = key - else: - assert cnt > 0 - of.write( - "{0} {1} {2} {3}\n".format( - last_name, last_offset, cnt, last_key - ) - ) - actdict[(last_name, last_key)] = last_offset, cnt - last_name, last_key = act.name, key - last_offset += offset_update_bytes - offset_update_bytes = 0 - cnt = 1 - else: - cnt += 1 - sf_line = f"{fmri} {act}\n" - sf.write(sf_line) - offset_update_bytes += len(sf_line.encode("utf-8")) - if last_name is not None: - assert last_key is not None - assert last_offset is not None - assert cnt > 0 - of.write( - "{0} {1} {2} {3}\n".format( - last_name, last_offset, cnt, last_key - ) - ) - actdict[(last_name, last_key)] = last_offset, cnt - - progtrack.job_add_progress(progtrack.JOB_FAST_LOOKUP) - - bad_keys = imageplan.ImagePlan._check_actions(nsd) - for k in sorted(bad_keys): - bf.write("{0}\n".format(k)) - - progtrack.job_add_progress(progtrack.JOB_FAST_LOOKUP) - sf.close() - of.close() - bf.close() - os.chmod(sp, misc.PKG_FILE_MODE) - os.chmod(op, misc.PKG_FILE_MODE) - os.chmod(bp, misc.PKG_FILE_MODE) - except BaseException as e: - try: - os.unlink(sp) - os.unlink(op) - os.unlink(bp) - except: - pass - raise - - progtrack.job_add_progress(progtrack.JOB_FAST_LOOKUP) + cache.rebuild(progtrack=progtrack) + except EnvironmentError as e: + if e.errno not in (errno.EACCES, errno.EROFS): + raise + self.__action_cache_dir = self.temporary_dir() + cache = actioncache.ActionCache(self, self.__action_cache_dir) + cache.rebuild(progtrack=progtrack) + self.__actioncache = cache + return cache + + def _sync_fast_lookups(self, progtrack=None): + """Bring the installed-action cache database into line with + the installed package catalog following an image-modifying + operation.""" - # Finally, rename the temporary files into their final place. - # If we have any problems, do our best to remove them, and we'll - # try to recreate them on the read-side. - try: - if not os.path.exists(self.__action_cache_dir): - os.makedirs(self.__action_cache_dir) - portable.rename(sp, stripped_path) - portable.rename(op, offsets_path) - portable.rename(bp, conflicting_keys_path) - except EnvironmentError as err: - if err.errno == errno.EACCES or err.errno == errno.EROFS: - self.__action_cache_dir = self.temporary_dir() - stripped_path = os.path.join( - self.__action_cache_dir, "actions.stripped" - ) - offsets_path = os.path.join( - self.__action_cache_dir, "actions.offsets" + cache = self.get_action_cache(progtrack=progtrack) + if DebugValues["actioncache-verify"]: + diff = cache.selfcheck() + if diff: + logger.error( + "WARNING: installed-action cache does not match " + "a full rebuild ({0}); rebuilding.".format(diff) ) - conflicting_keys_path = os.path.join( - self.__action_cache_dir, "keys.conflicting" - ) - portable.rename(sp, stripped_path) - portable.rename(op, offsets_path) - portable.rename(bp, conflicting_keys_path) - else: - try: - os.unlink(stripped_path) - os.unlink(offsets_path) - os.unlink(conflicting_keys_path) - except: - pass - raise err - - progtrack.job_add_progress(progtrack.JOB_FAST_LOOKUP) - progtrack.job_done(progtrack.JOB_FAST_LOOKUP) - return actdict, timestamp + cache = self._create_fast_lookups(progtrack=progtrack) + return cache def _remove_fast_lookups(self): - """Remove on-disk database created by _create_fast_lookups. - Should be called before updating image state to prevent the - client from seeing stale state if _create_fast_lookups is - interrupted.""" + """Remove the flat files that were used by older versions of + this code for the installed-action cache, so that an older + client sharing this image never trusts stale copies. The + sqlite database maintained by get_action_cache() is + deliberately left in place; it is reconciled against the + installed catalog on next use instead.""" for fname in ( "actions.stripped", @@ -3971,105 +3862,6 @@ def _remove_fast_lookups(self): continue raise apx._convert_error(e) - def _load_actdict(self, progtrack): - """Read the file of offsets created in _create_fast_lookups() - and return the dictionary mapping action name and key value to - offset.""" - - try: - of = open( - os.path.join(self.__action_cache_dir, "actions.offsets"), "r" - ) - except IOError as e: - if e.errno != errno.ENOENT: - raise - actdict, otimestamp = self._create_fast_lookups() - assert actdict is not None - self.__actdict = actdict - self.__actdict_timestamp = otimestamp - return actdict - - # Make sure the files are paired, and try to create them if not. - oversion = of.readline().rstrip() - otimestamp = of.readline().rstrip() - - # The original action.offsets file existed and had the same - # timestamp as the stored actdict, so that actdict can be - # reused. - if self.__actdict and otimestamp == self.__actdict_timestamp: - return self.__actdict - - sversion, stimestamp = self._get_stripped_actions_file(internal=True) - - # If we recognize neither file's version or their timestamps - # don't match, then we blow them away and try again. - if ( - oversion != "VERSION 2" - or sversion != "VERSION 1" - or stimestamp != otimestamp - ): - of.close() - actdict, otimestamp = self._create_fast_lookups() - assert actdict is not None - self.__actdict = actdict - self.__actdict_timestamp = otimestamp - return actdict - - # At this point, the original actions.offsets file existed, no - # actdict was saved in the image, the versions matched what was - # expected, and the timestamps of the actions.offsets and - # actions.stripped files matched, so the actions.offsets file is - # parsed to generate actdict. - actdict = {} - - for line in of: - actname, offset, cnt, key_attr = line.rstrip().split(None, 3) - off = int(offset) - actdict[(actname, key_attr)] = (off, int(cnt)) - - # This is a tight loop, so try to avoid burning - # CPU calling into the progress tracker excessively. - # Since we are already using the offset, we use that - # to damp calls back into the progress tracker. - if off % 500 == 0: - progtrack.plan_add_progress(progtrack.PLAN_ACTION_CONFLICT) - - of.close() - self.__actdict = actdict - self.__actdict_timestamp = otimestamp - return actdict - - def _get_stripped_actions_file(self, internal=False): - """Open the actions file described in _create_fast_lookups() and - return the corresponding file object.""" - - sf = open( - os.path.join(self.__action_cache_dir, "actions.stripped"), "r" - ) - sversion = sf.readline().rstrip() - stimestamp = sf.readline().rstrip() - if internal: - sf.close() - return sversion, stimestamp - - return sf - - def _load_conflicting_keys(self): - """Load the list of keys which have conflicting actions in the - existing image. If no such list exists, then return None.""" - - pth = os.path.join(self.__action_cache_dir, "keys.conflicting") - try: - with open(pth, "r") as fh: - version = fh.readline().rstrip() - if version != "VERSION 1": - return None - return set(l.rstrip() for l in fh) - except EnvironmentError as e: - if e.errno == errno.ENOENT: - return None - raise - def gen_installed_actions_bytype(self, atype, implicit_dirs=False): """Iterates through the installed actions of type 'atype'. If 'implicit_dirs' is True and 'atype' is 'dir', then include diff --git a/src/modules/client/imageplan.py b/src/modules/client/imageplan.py index a81271a30..aae8abee8 100644 --- a/src/modules/client/imageplan.py +++ b/src/modules/client/imageplan.py @@ -26,11 +26,9 @@ # from collections import defaultdict, namedtuple -import contextlib import errno import fnmatch import itertools -import mmap import operator import os import shutil @@ -3232,30 +3230,24 @@ def __update_act( keys, tgt, skip_dups, - offset_dict, + cache, action_classes, - sf, skip_fmris, fmri_dict, ): - """Update 'tgt' with action/fmri pairs from the stripped + """Update 'tgt' with action/fmri pairs from the installed action cache that are associated with the specified action 'keys'. The 'skip_dups' parameter indicates if we should avoid adding duplicate action/pfmri pairs into 'tgt'. - The 'offset_dict' parameter contains a mapping from key to - offsets into the actions.stripped file and the number of lines - to read. + The 'cache' parameter is the image's installed-action cache + (see pkg.client.actioncache). The 'action_classes' parameter contains the list of action types where one action can conflict with another action. - The 'sf' parameter is the actions.stripped file from which we - read the actual actions indicated by the offset dictionary - 'offset_dict.' - The 'skip_fmris' parameter contains a set of strings representing the packages which we should not process actions for. @@ -3264,54 +3256,33 @@ def __update_act( objects which is used so the same string isn't translated into the same PkgFmri object multiple times.""" - for key in keys: - offsets = [] - for klass in action_classes: - offset = offset_dict.get((klass.name, key), None) - if offset is not None: - offsets.append(offset) - - for offset, cnt in offsets: - sf.seek(offset) - pns = None - i = 0 - while 1: - # sf is reading in binary mode - line = misc.force_str(sf.readline()) - i += 1 - if i > cnt: - break - line = line.rstrip() - if line == "": - break - fmristr, actstr = line.split(None, 1) - if fmristr in skip_fmris: - continue - act = pkg.actions.fromstr(actstr) - if act.attrs[act.key_attr] != key: - raise api_errors.InvalidPackageErrors( - [ - "{} has invalid manifest " - "line:".format(fmristr), - " '{}'".format(actstr), - " '{}' vs. '{}'".format( - act.attrs[act.key_attr], key - ), - ] - ) - assert pns is None or act.namespace_group == pns - pns = act.namespace_group + anames = [klass.name for klass in action_classes] + pns = None + for key, fmristr, actstr in cache.get_actions(anames, keys): + if fmristr in skip_fmris: + continue + act = pkg.actions.fromstr(actstr) + if act.attrs[act.key_attr] != key: + raise api_errors.InvalidPackageErrors( + [ + "{} has invalid manifest line:".format(fmristr), + " '{}'".format(actstr), + " '{}' vs. '{}'".format( + act.attrs[act.key_attr], key + ), + ] + ) + assert pns is None or act.namespace_group == pns + pns = act.namespace_group - try: - pfmri = fmri_dict[fmristr] - except KeyError: - pfmri = pkg.fmri.PkgFmri(fmristr) - fmri_dict[fmristr] = pfmri - if skip_dups and self.__act_dup_check( - tgt, key, actstr, fmristr - ): - continue - tgt.setdefault(key, []).append((act, pfmri)) + try: + pfmri = fmri_dict[fmristr] + except KeyError: + pfmri = pkg.fmri.PkgFmri(fmristr) + fmri_dict[fmristr] = pfmri + if skip_dups and self.__act_dup_check(tgt, key, actstr, fmristr): + continue + tgt.setdefault(key, []).append((act, pfmri)) def __fast_check(self, new, old, ns): """Check whether actions being added and removed are @@ -3516,44 +3487,37 @@ def __check_conflicts(self, new, old, action_classes, ns, errs): ): continue + @staticmethod + def _check_action_group(ns, actions): + """Return True if the action/pfmri pairs in 'actions', which + all deliver to the same key attribute value within the + namespace group 'ns', conflict with each other.""" + + if len(actions) == 1: + return False + if ( + type(ns) != int + and ImagePlan.__check_inconsistent_types(actions, []) is not None + ): + return True + entry = actions[0][0] + if not entry.refcountable and entry.globally_identical: + return ImagePlan.__check_duplicate_actions(actions, []) is not None + if entry.globally_identical: + return ImagePlan.__check_inconsistent_attrs(actions, []) is not None + return False + @staticmethod def _check_actions(nsd): """Return the keys in the namespace dictionary ('nsd') which map to actions that conflict with each other.""" - def noop(*args): - return None - - bad_keys = set() - for ns, key_dict in nsd.items(): - if type(ns) != int: - type_func = ImagePlan.__check_inconsistent_types - else: - type_func = noop - for key, actions in key_dict.items(): - if len(actions) == 1: - continue - if type_func(actions, []) is not None: - bad_keys.add(key) - continue - if ( - not actions[0][0].refcountable - and actions[0][0].globally_identical - ): - if ( - ImagePlan.__check_duplicate_actions(actions, []) - is not None - ): - bad_keys.add(key) - continue - elif ( - actions[0][0].globally_identical - and ImagePlan.__check_inconsistent_attrs(actions, []) - is not None - ): - bad_keys.add(key) - continue - return bad_keys + return set( + key + for ns, key_dict in nsd.items() + for key, actions in key_dict.items() + if ImagePlan._check_action_group(ns, actions) + ) def __clear_pkg_plans(self): """Now that we're done reading the manifests, we can clear them @@ -3640,11 +3604,13 @@ def key(a): ) pt.plan_add_progress(pt.PLAN_ACTION_CONFLICT) - # Load information about the actions currently on the system. - offset_dict = self.image._load_actdict(self.__progtrack) - sf = self.image._get_stripped_actions_file() + # Load information about the actions currently on the system, + # without a progress tracker: any cache reconciliation happens + # inside this planning phase, whose progress rendering the + # fast-lookup job output would corrupt. + cache = self.image.get_action_cache() - conflict_clean_image = self.image._load_conflicting_keys() == set() + conflict_clean_image = not cache.has_conflicts() fmri_dict = weakref.WeakValueDictionary() # Iterate over action types in namespace groups first; our first @@ -3677,50 +3643,40 @@ def key(a): if conflict_clean_image: self.__fast_check(new, old, ns) - with contextlib.closing( - mmap.mmap(sf.fileno(), 0, access=mmap.ACCESS_READ) - ) as msf: - # Skip file header. - msf.readline() - msf.readline() - - # Update 'old' with all actions from the action - # cache which could conflict with the new - # actions being installed, or with actions - # already installed, but not getting removed. - keys = set(itertools.chain(new.keys(), old.keys())) - self.__update_act( - keys, - old, - False, - offset_dict, - action_classes, - msf, - gone_fmris, - fmri_dict, - ) + # Update 'old' with all actions from the action + # cache which could conflict with the new + # actions being installed, or with actions + # already installed, but not getting removed. + keys = set(itertools.chain(new.keys(), old.keys())) + self.__update_act( + keys, + old, + False, + cache, + action_classes, + gone_fmris, + fmri_dict, + ) - # Now update 'new' with all actions from the - # action cache which are staying on the system, - # and could conflict with the actions being - # installed. - keys = set(old.keys()) - self.__update_act( - keys, - new, - True, - offset_dict, - action_classes, - msf, - gone_fmris | changing_fmris, - fmri_dict, - ) + # Now update 'new' with all actions from the + # action cache which are staying on the system, + # and could conflict with the actions being + # installed. + keys = set(old.keys()) + self.__update_act( + keys, + new, + True, + cache, + action_classes, + gone_fmris | changing_fmris, + fmri_dict, + ) self.__check_conflicts(new, old, action_classes, ns, errs) del fmri_dict self.__clear_pkg_plans() - sf.close() self.__evaluate_fixups() pt.plan_done(pt.PLAN_ACTION_CONFLICT) @@ -5991,7 +5947,7 @@ def execute(self): else: self.pd._actuators.exec_post_actuators(self.image) - self.image._create_fast_lookups(progtrack=self.__progtrack) + self.image._sync_fast_lookups(progtrack=self.__progtrack) self.__save_release_notes() # success diff --git a/src/pkg/manifests/package:pkg.p5m b/src/pkg/manifests/package:pkg.p5m index b1d8bf5eb..56353445a 100644 --- a/src/pkg/manifests/package:pkg.p5m +++ b/src/pkg/manifests/package:pkg.p5m @@ -79,6 +79,7 @@ file path=$(PYDIRVP)/pkg/cfgfiles.py file path=$(PYDIRVP)/pkg/choose.py dir path=$(PYDIRVP)/pkg/client file path=$(PYDIRVP)/pkg/client/__init__.py +file path=$(PYDIRVP)/pkg/client/actioncache.py file path=$(PYDIRVP)/pkg/client/actuator.py file path=$(PYDIRVP)/pkg/client/api.py file path=$(PYDIRVP)/pkg/client/api_errors.py diff --git a/src/tests/api/t_actioncache.py b/src/tests/api/t_actioncache.py new file mode 100644 index 000000000..94abff282 --- /dev/null +++ b/src/tests/api/t_actioncache.py @@ -0,0 +1,314 @@ +#!/usr/bin/python3 +# +# This file and its contents are supplied under the terms of the +# Common Development and Distribution License ("CDDL"), version 1.0. +# You may only use this file in accordance with the terms of version +# 1.0 of the CDDL. +# +# A full copy of the text of the CDDL should have accompanied this +# source. A copy of the CDDL is also available via the Internet at +# http://www.illumos.org/license/CDDL. +# + +# +# Copyright 2026 OmniOS Community Edition (OmniOSce) Association. +# + +from . import testutils + +if __name__ == "__main__": + testutils.setup_environment("../../../proto") +import pkg5unittest + +import os +import sqlite3 +import unittest + +import pkg.client.actioncache as actioncache +from pkg.client.debugvalues import DebugValues +from pkg.client.imageplan import ImagePlan + + +class TestActionCache(pkg5unittest.SingleDepotTestCase): + """Tests for the sqlite installed-action cache that replaced the + actions.stripped/actions.offsets/keys.conflicting flat files.""" + + persistent_setup = False + + amber10 = """ + open amber@1.0,5.11-0 + add dir mode=0755 owner=root group=bin path=etc + add file amber1 mode=0644 owner=root group=bin path=etc/amber1 + add link path=etc/amber-link target=amber1 + close """ + + bronze10 = """ + open bronze@1.0,5.11-0 + add dir mode=0755 owner=root group=bin path=etc + add file bronze1 mode=0644 owner=root group=bin path=etc/bronze1 + close """ + + bronze20 = """ + open bronze@2.0,5.11-0 + add dir mode=0755 owner=root group=bin path=etc + add file bronze1 mode=0644 owner=root group=bin path=etc/bronze1 + add file bronze2 mode=0644 owner=root group=bin path=etc/bronze2 + close """ + + # Delivers the same path as amber's etc/amber1 with different + # content and mode; installing both is a conflict. + clash10 = """ + open clash@1.0,5.11-0 + add dir mode=0755 owner=root group=bin path=etc + add file clash1 mode=0600 owner=root group=bin path=etc/amber1 + close """ + + varpkg10 = """ + open varpkg@1.0,5.11-0 + add dir mode=0755 owner=root group=bin path=var + add file vp1 mode=0644 owner=root group=bin path=var/vp \ + variant.opensolaris.zone=global + add file vp2 mode=0644 owner=root group=bin path=var/vp \ + variant.opensolaris.zone=nonglobal + close """ + + misc_files = ["amber1", "bronze1", "bronze2", "clash1", "vp1", "vp2"] + + def setUp(self): + pkg5unittest.SingleDepotTestCase.setUp(self) + self.make_misc_files(self.misc_files) + self.pkgsend_bulk( + self.rurl, + ( + self.amber10, + self.bronze10, + self.bronze20, + self.clash10, + self.varpkg10, + ), + ) + + @staticmethod + def __legacy_bad_keys(img): + """Compute conflicting keys the way the flat-file code used + to: build the full namespace dictionary from every installed + manifest and run ImagePlan._check_actions over it.""" + + nsd = {} + excludes = img.list_excludes() + for pfmri in img.gen_installed_pkgs(): + m = img.get_manifest(pfmri, ignore_excludes=True) + for act in m.gen_actions(excludes=excludes): + if not act.globally_identical: + continue + act.strip() + nsd.setdefault(act.namespace_group, {}) + nsd[act.namespace_group].setdefault(act.attrs[act.key_attr], []) + nsd[act.namespace_group][act.attrs[act.key_attr]].append( + (act, pfmri) + ) + return ImagePlan._check_actions(nsd) + + @staticmethod + def __dump(cache): + """Return the comparable content of a cache database as + (package fmris, action rows, conflict rows).""" + + con = sqlite3.connect(cache.pathname) + try: + pkgs = frozenset( + r[0] for r in con.execute("SELECT fmri FROM packages") + ) + rows = frozenset( + con.execute( + "SELECT p.fmri, a.aname, a.keyval, a.act" + " FROM actions a" + " JOIN packages p ON p.pkg_id = a.pkg_id" + ) + ) + conf = frozenset(con.execute("SELECT ns, keyval FROM conflicts")) + finally: + con.close() + return pkgs, rows, conf + + def __assert_cache_matches(self, img): + """Assert that the image's (incrementally maintained) action + cache is identical to one rebuilt from scratch, and that its + conflicts match the legacy computation.""" + + cache = img.get_action_cache() + self.assertTrue(cache.is_fresh()) + + ref = actioncache.ActionCache(img, img.temporary_dir()) + ref.rebuild() + try: + self.assertEqual(self.__dump(cache), self.__dump(ref)) + finally: + ref.close() + + self.assertEqual(cache.conflicting_keys(), self.__legacy_bad_keys(img)) + + # Note that api_obj.reset(), which runs after every executed + # operation, recreates the underlying Image object, so tests must + # re-fetch api_inst.img after each operation rather than holding + # on to a stale Image reference. + + def test_01_lifecycle(self): + """The cache tracks install, update and uninstall operations + and always matches a from-scratch rebuild.""" + + api_inst = self.image_create(self.rurl) + + self._api_install(api_inst, ["amber", "bronze@1.0"]) + img = api_inst.img + cache = img.get_action_cache() + self.assertTrue( + os.path.exists(cache.pathname), "actions.sqlite created" + ) + self.assertFalse(cache.has_conflicts()) + self.__assert_cache_matches(img) + + # The legacy flat files must not be left around where an + # older client could trust them. + cdir = os.path.dirname(cache.pathname) + for fname in ( + "actions.stripped", + "actions.offsets", + "keys.conflicting", + ): + self.assertFalse(os.path.exists(os.path.join(cdir, fname))) + + self._api_update(api_inst, pkgs_update=["bronze@2.0"]) + self.__assert_cache_matches(api_inst.img) + + self._api_uninstall(api_inst, ["amber"]) + self.__assert_cache_matches(api_inst.img) + + def test_02_stale_cache_reconciled(self): + """A cache that is out of step with the installed catalog + (for instance after a boot environment rollback) is repaired + incrementally on next use.""" + + api_inst = self.image_create(self.rurl) + self._api_install(api_inst, ["amber"]) + + cache = api_inst.img.get_action_cache() + path = cache.pathname + + # Install another package, then put the pre-install database + # back, simulating out-of-band modification. + saved = open(path, "rb").read() + self._api_install(api_inst, ["bronze@1.0"]) + cache.close() + with open(path, "wb") as fh: + fh.write(saved) + + img2 = self.get_img_api_obj().img + self.__assert_cache_matches(img2) + + def test_03_conflicts(self): + """Conflicting actions are recorded in the conflicts table + and maintained incrementally as packages come and go.""" + + DebugValues["broken-conflicting-action-handling"] = 1 + try: + api_inst = self.image_create(self.rurl) + + self._api_install(api_inst, ["amber", "clash"]) + img = api_inst.img + cache = img.get_action_cache() + self.assertTrue(cache.has_conflicts()) + self.assertTrue("etc/amber1" in cache.conflicting_keys()) + self.__assert_cache_matches(img) + + self._api_uninstall(api_inst, ["clash"]) + img = api_inst.img + cache = img.get_action_cache() + self.assertFalse(cache.has_conflicts()) + self.__assert_cache_matches(img) + finally: + DebugValues.pop("broken-conflicting-action-handling", None) + + def test_04_variant_change(self): + """Changing variants changes the excludes signature and + forces a full rebuild with the new excludes.""" + + variants = {"variant.opensolaris.zone": "global"} + api_inst = self.image_create(self.rurl, variants=variants) + + self._api_install(api_inst, ["varpkg"]) + img = api_inst.img + cache = img.get_action_cache() + rows = self.__dump(cache)[1] + acts = [r[3] for r in rows if r[2] == "var/vp"] + self.assertEqual(len(acts), 1) + global_act = acts[0] + self.__assert_cache_matches(img) + + self._api_change_varcets( + api_inst, + variants={"variant.opensolaris.zone": "nonglobal"}, + ) + img = api_inst.img + cache = img.get_action_cache() + rows = self.__dump(cache)[1] + acts = [r[3] for r in rows if r[2] == "var/vp"] + self.assertEqual(len(acts), 1) + # The cache must now hold the nonglobal variant of the file. + self.assertNotEqual(acts[0], global_act) + self.__assert_cache_matches(img) + + def test_05_missing_and_corrupt(self): + """A missing or corrupt database is rebuilt transparently.""" + + api_inst = self.image_create(self.rurl) + self._api_install(api_inst, ["amber"]) + + cache = api_inst.img.get_action_cache() + path = cache.pathname + cache.close() + + os.unlink(path) + img2 = self.get_img_api_obj().img + self.__assert_cache_matches(img2) + self.assertTrue(os.path.exists(path)) + + with open(path, "r+b") as fh: + fh.seek(100) + fh.write(b"garbage" * 300) + img3 = self.get_img_api_obj().img + self.__assert_cache_matches(img3) + + def test_06_verify_debug(self): + """With -D actioncache-verify=1 set, a database that has + drifted from the installed manifests (here: tampered rows for + a package the operation doesn't touch, which an ordinary + reconcile would preserve) is detected after the operation and + rebuilt.""" + + api_inst = self.image_create(self.rurl) + self._api_install(api_inst, ["amber"]) + + cache = api_inst.img.get_action_cache() + self.assertIsNone(cache.selfcheck()) + path = cache.pathname + cache.close() + + con = sqlite3.connect(path) + con.execute("DELETE FROM actions WHERE aname = 'link'") + con.commit() + con.close() + + DebugValues["actioncache-verify"] = 1 + try: + self._api_install(api_inst, ["bronze@1.0"]) + finally: + DebugValues.pop("actioncache-verify", None) + + cache = api_inst.img.get_action_cache() + self.assertIsNone(cache.selfcheck()) + self.__assert_cache_matches(api_inst.img) + + +if __name__ == "__main__": + unittest.main()