Merge pull request #48 from networktocode/gfm-progress-bar

Add progress callback and example

Author: glennmatthews

README.md

@@ -22,9 +22,11 @@ A.sync_from(B)
 A.sync_to(B)
 ```
 
+You may wish to peruse the [`diffsync` GitHub topic](https://github.com/topics/diffsync) for examples of projects using this library.
+
 # Getting started
 
-To be able to properly compare different datasets, DiffSync relies on a shared datamodel that both systems must use.
+To be able to properly compare different datasets, DiffSync relies on a shared data model that both systems must use.
 Specifically, each system or dataset must provide a `DiffSync` "adapter" subclass, which in turn represents its dataset as instances of one or more `DiffSyncModel` data model classes.
 
 When comparing two systems, DiffSync detects the intersection between the two systems (which data models they have in common, and which attributes are shared between each pair of data models) and uses this intersection to compare and/or synchronize the data.
@@ -39,9 +41,9 @@ Each `DiffSyncModel` subclass supports the following class-level attributes:
 - `_attributes` - List of non-identifier instance field names for this object; used to identify the fields in common between data models for different systems (Optional)
 - `_children` - Dict of `{<model_name>: <field_name>}` indicating which fields store references to child data model instances. (Optional)
 
-> DiffSyncModel instances must be uniquely identified by their unique id, composed of all fields defined in `_identifiers`. The unique id must be globally meaningful (such as an unique instance name or slug), as it is used to identify object correspondence between differing systems or data sets. It **must not** be a value that is only locally meaningful, such as a database primary key integer value.
+> DiffSyncModel instances must be uniquely identified by their unique ID (or, in database terminology, [natural key](https://en.wikipedia.org/wiki/Natural_key)), which is composed of the union of all fields defined in `_identifiers`. The unique ID must be globally meaningful (such as an unique instance name or slug), as it is used to identify object correspondence between differing systems or data sets. It **must not** be a value that is only locally meaningful to a specific data set, such as a database primary key value.
 
-> Only fields listed in `_identifiers`, `_attributes`, or `_children` will be potentially included in comparison and synchronization between systems or data sets. Any other fields will be ignored; this allows for a model to additionally contain fields that are only locally relevant (such as database primary key values) and therefore are irrelevant to comparisons.
+> Only fields listed in `_identifiers`, `_attributes`, or `_children` will be potentially included in comparison and synchronization between systems or data sets. Any other fields will be ignored; this allows for a model to additionally contain fields that are only locally relevant (such as database primary key values) and therefore are irrelevant to comparison and synchronization.
 
 ```python
 from typing import List, Optional

diffsync/__init__.py

@@ -1,6 +1,6 @@
 """DiffSync front-end classes and logic.
 
-Copyright (c) 2020 Network To Code, LLC <info@networktocode.com>
+Copyright (c) 2020-2021 Network To Code, LLC <info@networktocode.com>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@
 """
 from collections import defaultdict
 from inspect import isclass
-from typing import ClassVar, Dict, List, Mapping, MutableMapping, Optional, Text, Tuple, Type, Union
+from typing import Callable, ClassVar, Dict, List, Mapping, MutableMapping, Optional, Text, Tuple, Type, Union
 
 from pydantic import BaseModel, PrivateAttr
 import structlog  # type: ignore
@@ -359,7 +359,7 @@ def remove_child(self, child: "DiffSyncModel"):
 class DiffSync:
     """Class for storing a group of DiffSyncModel instances and diffing/synchronizing to another DiffSync instance."""
 
-    # Add mapping of names to specific model classes here:
+    # In any subclass, you would add mapping of names to specific model classes here:
     # modelname1 = MyModelClass1
     # modelname2 = MyModelClass2
 
@@ -418,6 +418,10 @@ def __str__(self):
     def __repr__(self):
         return f"<{str(self)}>"
 
+    def __len__(self):
+        """Total number of elements stored in self._data."""
+        return sum(len(entries) for entries in self._data.values())
+
     def load(self):
         """Load all desired data from whatever backend data source into this instance."""
         # No-op in this generic class
@@ -451,29 +455,45 @@ def str(self, indent: int = 0) -> str:
     # Synchronization between DiffSync instances
     # ------------------------------------------------------------------------------
 
-    def sync_from(self, source: "DiffSync", diff_class: Type[Diff] = Diff, flags: DiffSyncFlags = DiffSyncFlags.NONE):
+    def sync_from(
+        self,
+        source: "DiffSync",
+        diff_class: Type[Diff] = Diff,
+        flags: DiffSyncFlags = DiffSyncFlags.NONE,
+        callback: Optional[Callable[[Text, int, int], None]] = None,
+    ):
         """Synchronize data from the given source DiffSync object into the current DiffSync object.
 
         Args:
             source (DiffSync): object to sync data from into this one
             diff_class (class): Diff or subclass thereof to use to calculate the diffs to use for synchronization
             flags (DiffSyncFlags): Flags influencing the behavior of this sync.
+            callback (function): Function with parameters (stage, current, total), to be called at intervals as the
+                calculation of the diff and subsequent sync proceed.
         """
-        diff = self.diff_from(source, diff_class=diff_class, flags=flags)
-        syncer = DiffSyncSyncer(diff=diff, src_diffsync=source, dst_diffsync=self, flags=flags)
+        diff = self.diff_from(source, diff_class=diff_class, flags=flags, callback=callback)
+        syncer = DiffSyncSyncer(diff=diff, src_diffsync=source, dst_diffsync=self, flags=flags, callback=callback)
         result = syncer.perform_sync()
         if result:
             self.sync_complete(source, diff, flags, syncer.base_logger)
 
-    def sync_to(self, target: "DiffSync", diff_class: Type[Diff] = Diff, flags: DiffSyncFlags = DiffSyncFlags.NONE):
+    def sync_to(
+        self,
+        target: "DiffSync",
+        diff_class: Type[Diff] = Diff,
+        flags: DiffSyncFlags = DiffSyncFlags.NONE,
+        callback: Optional[Callable[[Text, int, int], None]] = None,
+    ):
         """Synchronize data from the current DiffSync object into the given target DiffSync object.
 
         Args:
             target (DiffSync): object to sync data into from this one.
             diff_class (class): Diff or subclass thereof to use to calculate the diffs to use for synchronization
             flags (DiffSyncFlags): Flags influencing the behavior of this sync.
+            callback (function): Function with parameters (stage, current, total), to be called at intervals as the
+                calculation of the diff and subsequent sync proceed.
         """
-        target.sync_from(self, diff_class=diff_class, flags=flags)
+        target.sync_from(self, diff_class=diff_class, flags=flags, callback=callback)
 
     def sync_complete(
         self,
@@ -502,29 +522,43 @@ def sync_complete(
     # ------------------------------------------------------------------------------
 
     def diff_from(
-        self, source: "DiffSync", diff_class: Type[Diff] = Diff, flags: DiffSyncFlags = DiffSyncFlags.NONE
+        self,
+        source: "DiffSync",
+        diff_class: Type[Diff] = Diff,
+        flags: DiffSyncFlags = DiffSyncFlags.NONE,
+        callback: Optional[Callable[[Text, int, int], None]] = None,
     ) -> Diff:
         """Generate a Diff describing the difference from the other DiffSync to this one.
 
         Args:
             source (DiffSync): Object to diff against.
             diff_class (class): Diff or subclass thereof to use for diff calculation and storage.
             flags (DiffSyncFlags): Flags influencing the behavior of this diff operation.
+            callback (function): Function with parameters (stage, current, total), to be called at intervals as the
+                calculation of the diff proceeds.
         """
-        differ = DiffSyncDiffer(src_diffsync=source, dst_diffsync=self, flags=flags, diff_class=diff_class)
+        differ = DiffSyncDiffer(
+            src_diffsync=source, dst_diffsync=self, flags=flags, diff_class=diff_class, callback=callback
+        )
         return differ.calculate_diffs()
 
     def diff_to(
-        self, target: "DiffSync", diff_class: Type[Diff] = Diff, flags: DiffSyncFlags = DiffSyncFlags.NONE
+        self,
+        target: "DiffSync",
+        diff_class: Type[Diff] = Diff,
+        flags: DiffSyncFlags = DiffSyncFlags.NONE,
+        callback: Optional[Callable[[Text, int, int], None]] = None,
     ) -> Diff:
         """Generate a Diff describing the difference from this DiffSync to another one.
 
         Args:
             target (DiffSync): Object to diff against.
             diff_class (class): Diff or subclass thereof to use for diff calculation and storage.
             flags (DiffSyncFlags): Flags influencing the behavior of this diff operation.
+            callback (function): Function with parameters (stage, current, total), to be called at intervals as the
+                calculation of the diff proceeds.
         """
-        return target.diff_from(self, diff_class=diff_class, flags=flags)
+        return target.diff_from(self, diff_class=diff_class, flags=flags, callback=callback)
 
     # ------------------------------------------------------------------------------
     # Object Storage Management
@@ -567,21 +601,21 @@ def get(
             raise ObjectNotFound(f"{modelname} {uid} not present in {self.name}")
         return self._data[modelname][uid]
 
-    def get_all(self, obj: Union[Text, DiffSyncModel, Type[DiffSyncModel]]):
+    def get_all(self, obj: Union[Text, DiffSyncModel, Type[DiffSyncModel]]) -> List[DiffSyncModel]:
         """Get all objects of a given type.
 
         Args:
             obj: DiffSyncModel class or instance, or modelname string, that defines the type of the objects to retrieve
 
         Returns:
-            ValuesList[DiffSyncModel]: List of Object
+            List[DiffSyncModel]: List of Object
         """
         if isinstance(obj, str):
             modelname = obj
         else:
             modelname = obj.get_type()
 
-        return self._data[modelname].values()
+        return list(self._data[modelname].values())
 
     def get_by_uids(
         self, uids: List[Text], obj: Union[Text, DiffSyncModel, Type[DiffSyncModel]]

diffsync/diff.py

@@ -1,6 +1,6 @@
 """Diff and DiffElement classes for DiffSync.
 
-Copyright (c) 2020 Network To Code, LLC <info@networktocode.com>
+Copyright (c) 2020-2021 Network To Code, LLC <info@networktocode.com>
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -33,6 +33,13 @@ def __init__(self):
         `self.children[group][unique_id] == DiffElement(...)`
         """
 
+    def __len__(self):
+        """Total number of DiffElements stored herein."""
+        total = 0
+        for child in self.get_children():
+            total += len(child)
+        return total
+
     def complete(self):
         """Method to call when this Diff has been fully populated with data and is "complete".
 
@@ -205,6 +212,13 @@ def __str__(self):
             f"{self.source_name} → {self.dest_name} : {self.get_attrs_diffs()}"
         )
 
+    def __len__(self):
+        """Total number of DiffElements in this one, including itself."""
+        total = 1  # self
+        for child in self.get_children():
+            total += len(child)
+        return total
+
     @property
     def action(self) -> Optional[Text]:
         """Action, if any, that should be taken to remediate the diffs described by this element.