Merge pull request #81 from chadell/example-peeringdb

dgarros · web-flow · commit 53d6095b1a67 · 2021-12-14T11:01:14.000-05:00
Example 5: PeeringDB to Nautobot example
diff --git a/examples/05-nautobot-peeringdb/README.md b/examples/05-nautobot-peeringdb/README.md
@@ -0,0 +1,66 @@
+# Example 5 - PeeringDB to Nautobot synchronisation
+
+## Context
+
+The goal of this example is to synchronize some data from [PeeringDB](https://www.peeringdb.com/), that as the name suggests is a DB where peering entities define their facilities and presence to facilitate peering, towards [Nautobot Demo](https://demo.nautobot.com/) that is a always on demo service for [Nautobot](https://nautobot.readthedocs.io/), an open source Source of Truth.
+
+In Peering DB there is a model that defines a `Facility` and you can get information about the actual data center and the city where it is placed. In Nautobot, this information could be mapped to the `Region` and `Site` models, where `Region` can define hierarchy. For instance, Barcelona is in Spain and Spain is in Europe, and all of them are `Regions`. And, finally, the actual datacenter will refer to the `Region` where it is placed.
+
+Because of the nature of the demo, we will focus on syncing from PeeringDB to Nautobot (we can assume that PeeringDB is the authoritative System of Record) and we will skip the `delete` part of the `diffsync` library.
+
+We have 3 files:
+
+- `models.py`: defines the reference models that we will use: `RegionMode` and `SiteModel`
+- `adapter_peeringdb.py`: defines the PeeringDB adapter to translate via `load()` the data from PeeringDB into the reference models commented above. Notice that we don't define CRUD methods because we will sync from it (no to it)
+- `adapter_nautobot.py`: deifnes the Nautobot adapter with the `load()` and the CRUD methods
+
+> The source code for this example is in Github in the [examples/05-nautobot-peeringdb/](https://github.com/networktocode/diffsync/tree/main/examples/05-nautobot-peeringdb) directory.
+
+## Install dependencies
+
+```bash
+python3 -m venv .venv
+source .venv/bin/activate
+pip3 install -r requirements.txt
+```
+
+## Run it interactively
+
+```python
+from IPython import embed
+embed(colors="neutral")
+
+# Import Adapters
+from diffsync.enum import DiffSyncFlags
+
+from adapter_nautobot import NautobotRemote
+from adapter_peeringdb import PeeringDB
+
+# Initialize PeeringDB adapter, using CATNIX id for demonstration
+peeringdb = PeeringDB(ix_id=62)
+
+# Initialize Nautobot adapter, pointing to the demo instance (it's also the default settings)
+nautobot = NautobotRemote(
+    url="https://demo.nautobot.com",
+    token="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+)
+
+# Load PeeringDB info into the adapter
+peeringdb.load()
+
+# We can check the data that has been imported, some as `site` and some as `region` (with the parent relationships)
+peeringdb.dict()
+
+# Load Nautobot info into the adapter
+nautobot.load()
+
+# Let's diffsync do it's magic
+diff = nautobot.diff_from(peeringdb)
+
+# Quick summary of the expected changes (remember that delete ones are dry-run)
+diff.summary()
+
+# Execute the synchronization
+nautobot.sync_from(peeringdb, flags=DiffSyncFlags.SKIP_UNMATCHED_DST)
+
+```
diff --git a/examples/05-nautobot-peeringdb/adapter_nautobot.py b/examples/05-nautobot-peeringdb/adapter_nautobot.py
@@ -0,0 +1,199 @@
+"""Diffsync adapter class for Nautobot."""
+# pylint: disable=import-error,no-name-in-module
+import os
+import requests
+from models import RegionModel, SiteModel
+from diffsync import DiffSync
+
+
+NAUTOBOT_URL = os.getenv("NAUTOBOT_URL", "https://demo.nautobot.com")
+NAUTOBOT_TOKEN = os.getenv("NAUTOBOT_TOKEN", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+
+
+class RegionNautobotModel(RegionModel):
+    """Implementation of Region create/update/delete methods for updating remote Nautobot data."""
+
+    @classmethod
+    def create(cls, diffsync, ids, attrs):
+        """Create a new Region record in remote Nautobot.
+
+        Args:
+            diffsync (NautobotRemote): DiffSync adapter owning this Region
+            ids (dict): Initial values for this model's _identifiers
+            attrs (dict): Initial values for this model's _attributes
+        """
+        data = {
+            "name": ids["name"],
+            "slug": attrs["slug"],
+        }
+        if attrs["description"]:
+            data["description"] = attrs["description"]
+        if attrs["parent_name"]:
+            data["parent"] = str(diffsync.get(diffsync.region, attrs["parent_name"]).pk)
+        diffsync.post("/api/dcim/regions/", data)
+        return super().create(diffsync, ids=ids, attrs=attrs)
+
+    def update(self, attrs):
+        """Update an existing Region record in remote Nautobot.
+
+        Args:
+            attrs (dict): Updated values for this record's _attributes
+        """
+        data = {}
+        if "slug" in attrs:
+            data["slug"] = attrs["slug"]
+        if "description" in attrs:
+            data["description"] = attrs["description"]
+        if "parent_name" in attrs:
+            if attrs["parent_name"]:
+                data["parent"] = str(self.get(self.region, attrs["parent_name"]).pk)
+            else:
+                data["parent"] = None
+        self.diffsync.patch(f"/api/dcim/regions/{self.pk}/", data)
+        return super().update(attrs)
+
+    def delete(self):  # pylint: disable= useless-super-delegation
+        """Delete an existing Region record from remote Nautobot."""
+        # self.diffsync.delete(f"/api/dcim/regions/{self.pk}/")
+        return super().delete()
+
+
+class SiteNautobotModel(SiteModel):
+    """Implementation of Site create/update/delete methods for updating remote Nautobot data."""
+
+    @classmethod
+    def create(cls, diffsync, ids, attrs):
+        """Create a new Site in remote Nautobot.
+
+        Args:
+            diffsync (NautobotRemote): DiffSync adapter owning this Site
+            ids (dict): Initial values for this model's _identifiers
+            attrs (dict): Initial values for this model's _attributes
+        """
+        diffsync.post(
+            "/api/dcim/sites/",
+            {
+                "name": ids["name"],
+                "slug": attrs["slug"],
+                "description": attrs["description"],
+                "status": attrs["status_slug"],
+                "region": {"name": attrs["region_name"]} if attrs["region_name"] else None,
+                "latitude": attrs["latitude"],
+                "longitude": attrs["longitude"],
+            },
+        )
+        return super().create(diffsync, ids=ids, attrs=attrs)
+
+    def update(self, attrs):
+        """Update an existing Site record in remote Nautobot.
+
+        Args:
+            attrs (dict): Updated values for this record's _attributes
+        """
+        data = {}
+        if "slug" in attrs:
+            data["slug"] = attrs["slug"]
+        if "description" in attrs:
+            data["description"] = attrs["description"]
+        if "status_slug" in attrs:
+            data["status"] = attrs["status_slug"]
+        if "region_name" in attrs:
+            if attrs["region_name"]:
+                data["region"] = {"name": attrs["region_name"]}
+            else:
+                data["region"] = None
+        if "latitude" in attrs:
+            data["latitude"] = attrs["latitude"]
+        if "longitude" in attrs:
+            data["longitude"] = attrs["longitude"]
+        self.diffsync.patch(f"/api/dcim/sites/{self.pk}/", data)
+        return super().update(attrs)
+
+    def delete(self):  # pylint: disable= useless-super-delegation
+        """Delete an existing Site record from remote Nautobot."""
+        # self.diffsync.delete(f"/api/dcim/sites/{self.pk}/")
+        return super().delete()
+
+
+class NautobotRemote(DiffSync):
+    """DiffSync adapter class for loading data from a remote Nautobot instance using Python requests."""
+
+    # Model classes used by this adapter class
+    region = RegionNautobotModel
+    site = SiteNautobotModel
+
+    # Top-level class labels, i.e. those classes that are handled directly rather than as children of other models
+    top_level = ("region", "site")
+
+    def __init__(self, *args, url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN, **kwargs):
+        """Instantiate this class, but do not load data immediately from the remote system.
+
+        Args:
+            url (str): URL of the remote Nautobot system
+            token (str): REST API authentication token
+            job (Job): The running Job instance that owns this DiffSync adapter instance
+        """
+        super().__init__(*args, **kwargs)
+        if not url or not token:
+            raise ValueError("Both url and token must be specified!")
+        self.url = url
+        self.token = token
+        self.headers = {
+            "Accept": "application/json",
+            "Authorization": f"Token {self.token}",
+        }
+
+    def load(self):
+        """Load Region and Site data from the remote Nautobot instance."""
+        region_data = requests.get(f"{self.url}/api/dcim/regions/", headers=self.headers, params={"limit": 0}).json()
+        regions = region_data["results"]
+        while region_data["next"]:
+            region_data = requests.get(region_data["next"], headers=self.headers, params={"limit": 0}).json()
+            regions.extend(region_data["results"])
+
+        for region_entry in regions:
+            region = self.region(
+                name=region_entry["name"],
+                slug=region_entry["slug"],
+                description=region_entry["description"] or None,
+                parent_name=region_entry["parent"]["name"] if region_entry["parent"] else None,
+                pk=region_entry["id"],
+            )
+            self.add(region)
+
+        site_data = requests.get(f"{self.url}/api/dcim/sites/", headers=self.headers, params={"limit": 0}).json()
+        sites = site_data["results"]
+        while site_data["next"]:
+            site_data = requests.get(site_data["next"], headers=self.headers, params={"limit": 0}).json()
+            sites.extend(site_data["results"])
+
+        for site_entry in sites:
+            site = self.site(
+                name=site_entry["name"],
+                slug=site_entry["slug"],
+                status_slug=site_entry["status"]["value"] if site_entry["status"] else "active",
+                region_name=site_entry["region"]["name"] if site_entry["region"] else None,
+                description=site_entry["description"],
+                longitude=site_entry["longitude"],
+                latitude=site_entry["latitude"],
+                pk=site_entry["id"],
+            )
+            self.add(site)
+
+    def post(self, path, data):
+        """Send an appropriately constructed HTTP POST request."""
+        response = requests.post(f"{self.url}{path}", headers=self.headers, json=data)
+        response.raise_for_status()
+        return response
+
+    def patch(self, path, data):
+        """Send an appropriately constructed HTTP PATCH request."""
+        response = requests.patch(f"{self.url}{path}", headers=self.headers, json=data)
+        response.raise_for_status()
+        return response
+
+    def delete(self, path):
+        """Send an appropriately constructed HTTP DELETE request."""
+        response = requests.delete(f"{self.url}{path}", headers=self.headers)
+        response.raise_for_status()
+        return response
diff --git a/examples/05-nautobot-peeringdb/adapter_peeringdb.py b/examples/05-nautobot-peeringdb/adapter_peeringdb.py
@@ -0,0 +1,67 @@
+"""Diffsync adapter class for PeeringDB."""
+# pylint: disable=import-error,no-name-in-module
+import requests
+from slugify import slugify
+import pycountry
+from models import RegionModel, SiteModel
+from diffsync import DiffSync
+from diffsync.exceptions import ObjectNotFound
+
+
+PEERINGDB_URL = "https://peeringdb.com/"
+
+
+class PeeringDB(DiffSync):
+    """DiffSync adapter using requests to communicate with PeeringDB."""
+
+    # Model classes used by this adapter class
+    region = RegionModel
+    site = SiteModel
+
+    # Top-level class labels, i.e. those classes that are handled directly rather than as children of other models
+    top_level = ("region", "site")
+
+    def __init__(self, *args, ix_id, **kwargs):
+        """Initialize the PeeringDB adapter."""
+        super().__init__(*args, **kwargs)
+        self.ix_id = ix_id
+
+    def load(self):
+        """Load data via from PeeringDB."""
+        ix_data = requests.get(f"{PEERINGDB_URL}/api/ix/{self.ix_id}").json()
+
+        for fac in ix_data["data"][0]["fac_set"]:
+            # PeeringDB has no Region entity, so we must avoid duplicates
+            try:
+                self.get(self.region, fac["city"])
+            except ObjectNotFound:
+                # Use pycountry to translate the country code (like "DE") to a country name (like "Germany")
+                parent_name = pycountry.countries.get(alpha_2=fac["country"]).name
+                # Add the country as a parent region if not already added
+                try:
+                    self.get(self.region, parent_name)
+                except ObjectNotFound:
+                    parent_region = self.region(
+                        name=parent_name,
+                        slug=slugify(parent_name),
+                    )
+                    self.add(parent_region)
+
+                region = self.region(
+                    name=fac["city"],
+                    slug=slugify(fac["city"]),
+                    parent_name=parent_name,
+                )
+                self.add(region)
+
+            site = self.site(
+                name=fac["name"],
+                slug=slugify(fac["name"]),
+                status_slug="active",
+                region_name=fac["city"],
+                description=fac["notes"],
+                longitude=fac["longitude"],
+                latitude=fac["latitude"],
+                pk=fac["id"],
+            )
+            self.add(site)
diff --git a/examples/05-nautobot-peeringdb/main.py b/examples/05-nautobot-peeringdb/main.py
@@ -0,0 +1,32 @@
+"""Main.py."""
+
+# Import Adapters
+from adapter_nautobot import NautobotRemote
+from adapter_peeringdb import PeeringDB
+
+from diffsync.enum import DiffSyncFlags
+
+
+# Initialize PeeringDB adapter, using CATNIX id for demonstration
+peeringdb = PeeringDB(ix_id=62)
+
+# Initialize Nautobot adapter, pointing to the demo instance (it's also the default settings)
+nautobot = NautobotRemote(url="https://demo.nautobot.com", token="a" * 40)  # nosec
+
+# Load PeeringDB info into the adapter
+peeringdb.load()
+
+# We can check the data that has been imported, some as `site` and some as `region` (with the parent relationships)
+peeringdb.dict()
+
+# Load Nautobot info into the adapter
+nautobot.load()
+
+# Let's diffsync do it's magic
+diff = nautobot.diff_from(peeringdb)
+
+# Quick summary of the expected changes (remember that delete ones are dry-run)
+diff.summary()
+
+# Execute the synchronization
+nautobot.sync_from(peeringdb, flags=DiffSyncFlags.SKIP_UNMATCHED_DST)
diff --git a/examples/05-nautobot-peeringdb/models.py b/examples/05-nautobot-peeringdb/models.py
diff --git a/examples/05-nautobot-peeringdb/requirements.txt b/examples/05-nautobot-peeringdb/requirements.txt
