[FLOC-4421] Diffing of objects

flocker/control/_diffing.py

@@ -0,0 +1,247 @@
+# Copyright ClusterHQ Inc.  See LICENSE file for details.
+# -*- test-case-name: flocker.control.test.test_diffing -*-
+
+from pyrsistent import (
+    PClass,
+    PMap,
+    PVector,
+    PSet,
+    field,
+    freeze,
+    pvector,
+)
+
+from zope.interface import Interface, implementer
+
+
+class _DiffChange(Interface):
+    """
+    Interface for a diff change.
+
+    This is simply something that can be applied to an object to create a new
+    object.
+
+    This interface is created more to simply documentation than for any of the
+    actual zope.interface mechanisms.
+    """
+
+    def apply(obj):
+        """
+        Apply this diff change to the passed in object and return a new object
+        that is obj with the ``self`` diff applied.
+
+        :param object obj: The object to apply the diff to.
+
+        :returns: A new object that is the passed in object with the diff
+            applied.
+        """
+        pass
+
+
+@implementer(_DiffChange)
+class _Remove(PClass):
+    """
+    A ``_DiffChange`` that removes an object from a ``PSet`` or a key from a
+    ``PMap`` inside a nested object tree.
+
+    :ivar path: The path in the nested object tree of the object to be removed
+        from the import set.
+    """
+
+    path = field(
+        type=PVector,
+        factory=freeze
+    )
+
+    def apply(self, obj):
+        obj_path = self.path[:-1]
+        removal_path = self.path[-1]
+        return obj.transform(obj_path, lambda o: o.remove(removal_path))
+
+
+@implementer(_DiffChange)
+class _Set(PClass):
+    """
+    A ``_DiffChange`` that sets a field in a ``PClass`` or sets a key in a
+    ``PMap``.
+
+    :ivar path: The path in the nested object to the field/key to be set to a
+        new value.
+
+    :ivar val: The value to set the field/key to.
+    """
+    path = field(
+        type=PVector,
+        factory=freeze
+    )
+    val = field()
+
+    def apply(self, obj):
+        return obj.transform(self.path, self.val)
+
+
+@implementer(_DiffChange)
+class _Add(PClass):
+    """
+    A ``_DiffChange`` that adds an item to a ``PSet``.
+
+    :ivar path: The path to the set to which the item will be added.
+
+    :ivar item: The item to be added to the set.
+    """
+    path = field(
+        type=PVector,
+        factory=freeze
+    )
+    item = field()
+
+    def apply(self, obj):
+        return obj.transform(self.path, lambda x: x.add(self.item))
+
+
+@implementer(_DiffChange)
+class _Diff(PClass):
+    """
+    A ``_DiffChange`` that is simply the serial application of other diff
+    changes.
+
+    This is the object that external modules get and use to apply diffs to
+    objects.
+
+    :ivar changes: A vector of ``_DiffChange`` s that represent a diff between
+        two objects.
+    """
+
+    changes = field(
+        type=PVector,
+        factory=freeze
+    )
+
+    def apply(self, obj):
+        for c in self.changes:
+            obj = c.apply(obj)
+        return obj
+
+
+def _create_diffs_for_sets(current_path, set_a, set_b):
+    """
+    Computes a series of ``_DiffChange`` s to turn ``set_a`` into ``set_b``
+    assuming that these sets are at ``current_path`` inside a nested pyrsistent
+    object.
+
+    :param current_path: And iterable of pyrsistent object describing the path
+        inside the root pyrsistent object where the other arguments are
+        located.  See ``PMap.transform`` for the format of this sort of path.
+
+    :param set_a: The desired input set.
+
+    :param set_b: The desired output set.
+
+    :returns: An iterable of ``_DiffChange`` s that will turn ``set_a`` into
+        ``set_b``.
+    """
+    resulting_diffs = pvector([]).evolver()
+    for item in set_a.difference(set_b):
+        resulting_diffs.append(
+            _Remove(path=current_path.append(item))
+        )
+    for item in set_b.difference(set_a):
+        resulting_diffs.append(
+            _Add(path=current_path, item=item)
+        )
+    return resulting_diffs.persistent()
+
+
+def _create_diffs_for_mappings(current_path, mapping_a, mapping_b):
+    """
+    Computes a series of ``_DiffChange`` s to turn ``mapping_a`` into
+    ``mapping_b`` assuming that these mappings are at ``current_path`` inside a
+    nested pyrsistent object.
+
+    :param current_path: And iterable of pyrsistent object describing the path
+        inside the root pyrsistent object where the other arguments are
+        located.  See ``PMap.transform`` for the format of this sort of path.
+
+    :param mapping_a: The desired input mapping.
+
+    :param mapping_b: The desired output mapping.
+
+    :returns: An iterable of ``_DiffChange`` s that will turn ``mapping_a``
+        into ``mapping_b``.
+    """
+    resulting_diffs = pvector([]).evolver()
+    a_keys = frozenset(x for x in mapping_a.iterkeys())
+    b_keys = frozenset(x for x in mapping_b.iterkeys())
+    for key in a_keys.intersection(b_keys):
+        if mapping_a[key] != mapping_b[key]:
+            resulting_diffs.extend(
+                _create_diffs_for(
+                    current_path.append(key),
+                    mapping_a[key],
+                    mapping_b[key]
+                )
+            )
+    for key in b_keys.difference(a_keys):
+        resulting_diffs.append(
+            _Set(path=current_path.append(key), val=mapping_b[key])
+        )
+    for key in a_keys.difference(b_keys):
+        resulting_diffs.append(
+            _Remove(path=current_path.append(key))
+        )
+    return resulting_diffs.persistent()
+
+
+def _create_diffs_for(current_path, subobj_a, subobj_b):
+    """
+    Computes a series of ``_DiffChange`` s to turn ``subobj_a`` into
+    ``subobj_b`` assuming that these subobjs are at ``current_path`` inside a
+    nested pyrsistent object.
+
+    :param current_path: And iterable of pyrsistent object describing the path
+        inside the root pyrsistent object where the other arguments are
+        located.  See ``PMap.transform`` for the format of this sort of path.
+
+    :param subobj_a: The desired input sub object.
+
+    :param subobj_b: The desired output sub object.
+
+    :returns: An iterable of ``_DiffChange`` s that will turn ``subobj_a``
+        into ``subobj_b``.
+    """
+    if subobj_a == subobj_b:
+        return pvector([])
+    elif type(subobj_a) != type(subobj_b):
+        return pvector([_Set(path=current_path, val=subobj_b)])
+    elif isinstance(subobj_a, PClass) and isinstance(subobj_b, PClass):
+        a_dict = subobj_a._to_dict()
+        b_dict = subobj_b._to_dict()
+        return _create_diffs_for_mappings(current_path, a_dict, b_dict)
+    elif isinstance(subobj_a, PMap) and isinstance(subobj_b, PMap):
+        return _create_diffs_for_mappings(
+            current_path, subobj_a, subobj_b)
+    elif isinstance(subobj_a, PSet) and isinstance(subobj_b, PSet):
+        return _create_diffs_for_sets(
+            current_path, subobj_a, subobj_b)
+    return pvector([_Set(path=current_path, val=subobj_b)])
+
+
+def create_diff(object_a, object_b):
+    """
+    Constructs a diff from ``object_a`` to ``object_b``
+
+    :param object_a: The desired input object.
+
+    :param object_b: The desired output object.
+
+    :returns:  A ``_Diff`` that will convert ``object_a`` into ``object_b``
+        when applied.
+    """
+    changes = _create_diffs_for(pvector([]), object_a, object_b)
+    return _Diff(changes=changes)
+
+
+# Ensure that the representation of a ``_Diff`` is entirely serializable:
+DIFF_SERIALIZABLE_CLASSES = [
+    _Set, _Remove, _Add, _Diff
+]

flocker/control/_model.py

@@ -26,6 +26,8 @@
 
 from zope.interface import Interface, implementer
 
+from ._diffing import DIFF_SERIALIZABLE_CLASSES
+
 
 def _sequence_field(checked_class, suffix, item_type, optional, initial):
     """
@@ -1235,5 +1237,5 @@ def get_information_wipe(self):
     Deployment, Node, DockerImage, Port, Link, RestartNever, RestartAlways,
     RestartOnFailure, Application, Dataset, Manifestation, AttachedVolume,
     NodeState, DeploymentState, NonManifestDatasets, Configuration,
-    Lease, Leases, PersistentState,
-]
+    Lease, Leases, PersistentState
+] + DIFF_SERIALIZABLE_CLASSES

flocker/control/test/test_diffing.py

@@ -0,0 +1,123 @@
+# Copyright ClusterHQ Inc.  See LICENSE file for details.
+
+"""
+Tests for ``flocker.node._diffing``.
+"""
+
+from uuid import uuid4
+
+from hypothesis import given
+
+from .._diffing import create_diff
+from .._persistence import wire_encode, wire_decode
+from .._model import Node, Port
+from ..testtools import (
+    application_strategy,
+    deployment_strategy
+)
+
+from ...testtools import TestCase
+
+from testtools.matchers import Equals, LessThan
+
+
+class DeploymentDiffTest(TestCase):
+    """
+    Tests for creating and applying diffs between deployments.
+    """
+
+    @given(
+        deployment_strategy(),
+        deployment_strategy(),
+    )
+    def test_deployment_diffing(self, deployment_a, deployment_b):
+        """
+        Diffing two arbitrary deployments, then applying the diff to the first
+        deployment yields the second even after the diff has been serialized
+        and re-created.
+        """
+        diff = create_diff(deployment_a, deployment_b)
+        serialized_diff = wire_encode(diff)
+        newdiff = wire_decode(serialized_diff)
+        should_b_b = newdiff.apply(deployment_a)
+        self.assertThat(
+            should_b_b,
+            Equals(deployment_b)
+        )
+
+    def test_deployment_diffing_smart(self):
+        """
+        Small modifications to a deployment have diffs that are small. Their
+        reverse is also small.
+        """
+        # Any large deployment will do, just use hypothesis for convenience of
+        # generating a large deployment.
+        deployment = deployment_strategy(min_number_of_nodes=90).example()
+
+        new_nodes = list(Node(uuid=uuid4()) for _ in xrange(4))
+        d = reduce(lambda x, y: x.update_node(y), new_nodes, deployment)
+        encoded_deployment = wire_encode(deployment)
+
+        diff = create_diff(deployment, d)
+        encoded_diff = wire_encode(diff)
+        self.assertThat(
+            len(encoded_diff),
+            LessThan(len(encoded_deployment)/2)
+        )
+        self.assertThat(
+            wire_decode(encoded_diff).apply(deployment),
+            Equals(d)
+        )
+
+        removal_diff = create_diff(d, deployment)
+        encoded_removal_diff = wire_encode(removal_diff)
+        self.assertThat(
+            len(encoded_removal_diff),
+            LessThan(len(encoded_deployment)/2)
+        )
+        self.assertThat(
+            wire_decode(encoded_removal_diff).apply(d),
+            Equals(deployment)
+        )
+
+    def test_set_diffing_smart(self):
+        """
+        Small modifications to a sets have diffs that are small. Their reverse
+        is also small.
+        """
+        # Any Application with a large set of ports will do, just use
+        # hypothesis for convenience of generating a large number of ports on
+        # an application.
+        application = application_strategy(min_number_of_ports=1000).example()
+
+        new_ports = list(
+            Port(internal_port=i, external_port=i) for i in xrange(4)
+        )
+        a = reduce(
+            lambda x, y: x.transform(['ports'], lambda x: x.add(y)),
+            new_ports,
+            application
+        )
+        encoded_application = wire_encode(application)
+
+        diff = create_diff(application, a)
+        encoded_diff = wire_encode(diff)
+        self.assertThat(
+            len(encoded_diff),
+            LessThan(len(encoded_application)/2)
+        )
+        self.assertThat(
+            wire_decode(encoded_diff).apply(application),
+            Equals(a)
+        )
+
+        removal_diff = create_diff(a, application)
+        encoded_removal_diff = wire_encode(removal_diff)
+        self.assertThat(
+            len(encoded_removal_diff),
+            LessThan(len(encoded_application)/2)
+        )
+        self.assertThat(
+            wire_decode(encoded_removal_diff).apply(a),
+            Equals(application)
+        )