From b516bb542197b73c68810a2d90e5c79079c3c24e Mon Sep 17 00:00:00 2001 From: Mika Busch Date: Thu, 7 May 2020 17:11:30 +0200 Subject: [PATCH 01/10] Adding comments to vm sync vm.config.annotation is synced as comment to netbox --- run.py | 1 + templates/netbox.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/run.py b/run.py index 23601b2..4ef19e4 100644 --- a/run.py +++ b/run.py @@ -711,6 +711,7 @@ def get_objects(self, vc_obj_type): results["virtual_machines"].append(nbt.virtual_machine( name=truncate(obj_name, max_len=64), cluster=cluster, + comments=getattr(obj.config, "annotation", None), status=int( 1 if obj.runtime.powerState == "poweredOn" else 0 ), diff --git a/templates/netbox.py b/templates/netbox.py index b1eed35..ece2fcf 100644 --- a/templates/netbox.py +++ b/templates/netbox.py @@ -625,7 +625,7 @@ def virtual_machine(self, name, cluster, status=None, role=None, "vcpus": vcpus, "memory": memory, "disk": disk, - "comments": comments, + "comments": comments.replace("\n", "\r\n\r") if comments else None, "local_context_data": local_context_data, "tags": tags } From 21a71fd865919a92ba7410b58c995cf4b83b5ea9 Mon Sep 17 00:00:00 2001 From: Mika Busch Date: Wed, 17 Jun 2020 10:30:50 +0200 Subject: [PATCH 02/10] Use description instead of comments Tags use a description field instead of a comments field. This Field is also limited to 200 characters --- run.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/run.py b/run.py index 23601b2..8e60d7f 100644 --- a/run.py +++ b/run.py @@ -1550,9 +1550,8 @@ def verify_dependencies(self): "name": "Orphaned", "slug": "orphaned", "color": "607d8b", - "comments": "This applies to objects that have become " - "orphaned. The source system which has " - "previously provided the object no longer " + "description": "The source system which has previously" + "provided the object no longer " "states it exists.{}".format( " An object with the 'Orphaned' tag will " "remain in this state until it ages out " @@ -1562,14 +1561,14 @@ def verify_dependencies(self): { "name": self.vc_tag, "slug": format_slug(self.vc_tag), - "comments": "Objects synced from vCenter host " + "description": "Objects synced from vCenter host " "{}. Be careful not to modify the name or " "slug.".format(self.vc_tag) }, { "name": "vCenter", "slug": "vcenter", - "comment": "Created and used by vCenter NetBox sync." + "description": "Created and used by vCenter NetBox sync." }], "manufacturers": [ {"name": "VMware", "slug": "vmware"}, From 76e2efdf686365ceca3361c8dfd61c4b73541e9d Mon Sep 17 00:00:00 2001 From: Mika Busch Date: Mon, 22 Jun 2020 12:12:18 +0200 Subject: [PATCH 03/10] Fixed Pylinter errors --- run.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/run.py b/run.py index 8e60d7f..2c535be 100644 --- a/run.py +++ b/run.py @@ -1551,19 +1551,19 @@ def verify_dependencies(self): "slug": "orphaned", "color": "607d8b", "description": "The source system which has previously" - "provided the object no longer " - "states it exists.{}".format( - " An object with the 'Orphaned' tag will " - "remain in this state until it ages out " - "and is automatically removed." - ) if settings.NB_PRUNE_ENABLED else "" + "provided the object no longer " + "states it exists.{}".format( + " An object with the 'Orphaned' tag will " + "remain in this state until it ages out " + "and is automatically removed." + ) if settings.NB_PRUNE_ENABLED else "" }, { "name": self.vc_tag, "slug": format_slug(self.vc_tag), "description": "Objects synced from vCenter host " - "{}. Be careful not to modify the name or " - "slug.".format(self.vc_tag) + "{}. Be careful not to modify the name or " + "slug.".format(self.vc_tag) }, { "name": "vCenter", From f09cc9c0c110c7dd1e0088662065a1bddc4e9019 Mon Sep 17 00:00:00 2001 From: Mika Busch Date: Fri, 19 Jun 2020 01:33:18 +0200 Subject: [PATCH 04/10] Removed duplicate Logging same log statement already present at the start of the loop --- run.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/run.py b/run.py index 23601b2..5847f95 100644 --- a/run.py +++ b/run.py @@ -683,10 +683,6 @@ def get_objects(self, vc_obj_type): tags=self.tags, )) elif vc_obj_type == "virtual_machines": - log.info( - "Collecting info about vCenter %s '%s' object.", - vc_obj_type, obj_name - ) # Virtual Machines log.debug( "Collecting info for virtual machine '%s'", obj_name From d373b106e74d984e19cddca1a746e0b0b7e20e93 Mon Sep 17 00:00:00 2001 From: Mika Busch Date: Wed, 17 Jun 2020 10:34:44 +0200 Subject: [PATCH 05/10] Strip whitespaces when checking if asset tag is banned Sometimes vCenter returns asset tags with a leading whitespace. Instead of adding a second version with a whitespace to the banned tags list we use the strip() method to remove the whitespace bevore checking --- run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run.py b/run.py index 23601b2..2d38d1a 100644 --- a/run.py +++ b/run.py @@ -204,11 +204,11 @@ def is_banned_asset_tag(text): # Is asset tag in banned list? text = text.lower() banned_tags = [ - "Default string", "NA", "N/A", "None", " none", "Null", "oem", "o.e.m", + "Default string", "NA", "N/A", "None", "Null", "oem", "o.e.m", "to be filled by o.e.m.", "Unknown", " ", "" ] banned_tags = [t.lower() for t in banned_tags] - if text in banned_tags: + if text.strip() in banned_tags: result = True # Does it exceed the max allowed length for NetBox asset tags? elif len(text) > 50: From 3bee0e2d06d05ca1c91334d1151b609147227b79 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Wed, 16 Sep 2020 14:51:17 +0200 Subject: [PATCH 06/10] adds reattemps to netbox requests --- run.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/run.py b/run.py index dac47c6..609ccec 100644 --- a/run.py +++ b/run.py @@ -967,6 +967,27 @@ def get_primary_ip(self, nb_obj_type, nb_id): return result def request(self, req_type, nb_obj_type, data=None, query=None, nb_id=None): + + max_retry_attempts = 3 + + for _ in range(max_retry_attempts): + + try: + result = self.single_request(req_type, nb_obj_type, data, query, nb_id) + except (SystemExit, ConnectionError, requests.exceptions.ConnectionError, + requests.exceptions.ReadTimeout): + log.warning("Request failed, trying again.") + continue + else: + break + else: + raise SystemExit( + log.critical("Giving up after %d retries.", max_retry_attempts) + ) + + return result + + def single_request(self, req_type, nb_obj_type, data=None, query=None, nb_id=None): """ HTTP requests and exception handler for NetBox From 4d366b79d64203246d68400fcfd3520c4e042e3d Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Thu, 17 Sep 2020 12:17:16 +0200 Subject: [PATCH 07/10] increases python rsponse version to 2.24.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4b2a43f..0f48c82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ idna==2.8 pycares==3.1.1 pycparser==2.19 pyvmomi==6.7.3 -requests==2.22.0 +requests==2.24.0 six==1.13.0 typing==3.7.4.1 urllib3==1.25.7 From 069f81c0310ed341607133338a780b1135c225eb Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Thu, 17 Sep 2020 12:17:40 +0200 Subject: [PATCH 08/10] cleans up duplicate functions --- run.py | 40 +--------------------------------------- templates/netbox.py | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 42 deletions(-) diff --git a/run.py b/run.py index 609ccec..de041d3 100644 --- a/run.py +++ b/run.py @@ -14,7 +14,7 @@ from pyVmomi import vim import settings from logger import log -from templates.netbox import Templates +from templates.netbox import Templates, truncate, format_slug @@ -109,30 +109,6 @@ def format_ip(ip_addr): return result -def format_slug(text): - """ - Format string to comply to NetBox slug acceptable pattern and max length. - - :param text: Text to be formatted into an acceptable slug - :type text: str - :return: Slug of allowed characters [-a-zA-Z0-9_] with max length of 50 - :rtype: str - """ - allowed_chars = ( - "abcdefghijklmnopqrstuvxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" # Alphabet - "01234567890" # Numbers - "_-" # Symbols - ) - # Replace seperators with dash - seperators = [" ", ",", "."] - for sep in seperators: - text = text.replace(sep, "-") - # Strip unacceptable characters - text = "".join([c for c in text if c in allowed_chars]) - # Enforce max length - return truncate(text, max_len=50).lower() - - def format_tag(tag): """ Format string to comply to NetBox tag format and max length. @@ -323,20 +299,6 @@ async def reverse_lookup(resolver, ip): return result -def truncate(text="", max_len=50): - """ - Ensure a string complies to the maximum length specified. - - :param text: Text to be checked for length and truncated if necessary - :type text: str - :param max_len: Max length of the returned string - :type max_len: int, optional - :return: Text in :param text: truncated to :param max_len: if necessary - :rtype: str - """ - return text if len(text) < max_len else text[:max_len] - - def verify_ip(ip_addr): """ Verify input is expected format and checks against allowed networks. diff --git a/templates/netbox.py b/templates/netbox.py index ece2fcf..2350769 100644 --- a/templates/netbox.py +++ b/templates/netbox.py @@ -24,10 +24,10 @@ def format_slug(text, max_len=50): """ allowed_chars = ( "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" # Alphabet - "01234567890" # Numbers + "0123456789" # Numbers "_-" # Symbols ) - # Replace seperators with dash + # Replace separators with dash seperators = [" ", ",", "."] for sep in seperators: text = text.replace(sep, "-") @@ -38,7 +38,16 @@ def format_slug(text, max_len=50): def truncate(text="", max_len=50): - """Ensure a string complies to the maximum length specified.""" + """ + Ensure a string complies to the maximum length specified. + + :param text: Text to be checked for length and truncated if necessary + :type text: str + :param max_len: Max length of the returned string + :type max_len: int, optional + :return: Text in :param text: truncated to :param max_len: if necessary + :rtype: str + """ return text if len(text) < max_len else text[:max_len] From 2d3052fd14bc3947b24e1da3927e93e4e44b6355 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Thu, 17 Sep 2020 17:07:40 +0200 Subject: [PATCH 09/10] adds support for NetBox 2.9 changed tag behaviour --- run.py | 138 +++++++++++++++++++++++++++++++++++--------- templates/netbox.py | 34 +++++++++++ 2 files changed, 144 insertions(+), 28 deletions(-) diff --git a/run.py b/run.py index de041d3..08c1554 100644 --- a/run.py +++ b/run.py @@ -740,6 +740,9 @@ class NetBoxHandler: :param vc_conn: Connection details for a vCenter host defined in settings.py :type vc_conn: dict """ + + instance_tags = None + def __init__(self, vc_conn): self.nb_api_url = "http{}://{}{}/api/".format( ("s" if not settings.NB_DISABLE_TLS else ""), settings.NB_FQDN, @@ -1054,6 +1057,41 @@ def single_request(self, req_type, nb_obj_type, data=None, query=None, nb_id=Non ) return result + def update_tag_data(self, tag_list): + """ + updates missing tags in NetBox and reformats list of tags to list of tag dicts + + :param tag_list: list of tags + :type tag_list: list + :return: list of reformatted tags + :rtype: list + """ + nbt = Templates(api_version=self.nb_api_version) + + for tag in tag_list: + + if isinstance(tag, dict): + tag = tag.get("name") + + if tag is None: + continue + + if tag not in [d.get("name") for d in self.instance_tags]: + log.info( + "Netbox tag '%s' object not found. Requesting creation.", + tag, + ) + self.request( + req_type="post", + nb_obj_type="tags", + data=nbt.tag(name=tag) + ) + + # update tag information + self.get_instance_tags() + + return [{"name": d } for d in tag_list] + def obj_exists(self, nb_obj_type, vc_data): """ Checks whether a NetBox object exists and matches the vCenter object. @@ -1069,6 +1107,7 @@ def obj_exists(self, nb_obj_type, vc_data): # NetBox Device Types objects do not have names to query; we catch # and use the model instead query_key = self.obj_map[nb_obj_type]["key"] + # Create a query specific to the device parent/child relationship when # working with interfaces if nb_obj_type == "interfaces": @@ -1084,13 +1123,17 @@ def obj_exists(self, nb_obj_type, vc_data): query = "?{}={}".format( query_key, vc_data[query_key] ) + # Add support for filtering objects by vCenter tags if self.obj_map[nb_obj_type]["taggable"]: query = query + "&?tag={}".format(self.vc_tag) + + # request object from NetBox req = self.request( req_type="get", nb_obj_type=nb_obj_type, query=query ) + # A single matching object is found so we compare its values to the new # object if req["count"] == 1: @@ -1099,9 +1142,19 @@ def obj_exists(self, nb_obj_type, vc_data): nb_obj_type, vc_data[query_key] ) nb_data = req["results"][0] + + # reduce NetBox tags to names only + if nb_data.get("tags"): + nb_data["tags"] = [{"name": d["name"]} for d in nb_data["tags"]] + + # add missing tags before assigning them + if "tags" in vc_data: + + vc_data["tags"] = self.update_tag_data(vc_data["tags"]) + # Objects that have been previously tagged as orphaned but then # reappear in vCenter need to be stripped of their orphaned status - if "tags" in vc_data and "Orphaned" in nb_data["tags"]: + if "tags" in nb_data and "Orphaned" in [d.get("name") for d in nb_data["tags"]]: log.info( "NetBox %s object '%s' is currently marked as orphaned " "but has reappeared in vCenter. Updating NetBox.", @@ -1129,7 +1182,7 @@ def obj_exists(self, nb_obj_type, vc_data): if "tags" in vc_data: log.debug("Merging tags between vCenter and NetBox object.") vc_data["tags"] = list( - set(vc_data["tags"] + nb_data["tags"]) + {x['name']:x for x in vc_data["tags"] + nb_data["tags"]}.values() ) # Remove site from existing NetBox host objects to allow for # user modifications @@ -1151,6 +1204,10 @@ def obj_exists(self, nb_obj_type, vc_data): nb_obj_type, vc_data[query_key], req["count"] ) else: + + if "tags" in vc_data: + vc_data["tags"] = self.update_tag_data(vc_data["tags"]) + log.info( "Netbox %s '%s' object not found. Requesting creation.", nb_obj_type, @@ -1524,32 +1581,45 @@ def verify_dependencies(self): """ Validates that all prerequisite NetBox objects exist and creates them. """ + nbt = Templates(api_version=self.nb_api_version) + tag_dependencies = [ + nbt.tag( + name="Orphaned", + color="607d8b", + description="The source system which has previously" + "provided the object no longer " + "states it exists.{}".format( + " An object with the 'Orphaned' tag will " + "remain in this state until it ages out " + "and is automatically removed." + ) if settings.NB_PRUNE_ENABLED else "" + ), + nbt.tag( + name=self.vc_tag, + description="Objects synced from vCenter host " + "{}. Be careful not to modify the name or " + "slug.".format(self.vc_tag) + ), + nbt.tag( + name="vCenter", + description="Created and used by vCenter NetBox sync." + ), + nbt.tag( + name="Synced", + description="Created and used by vCenter NetBox sync." + ) + ] + + log.info("Creating Tags if not present") + for tag in tag_dependencies: + self.obj_exists(nb_obj_type="tags", vc_data=tag) + + log.info("Finished creating tags.") + + # retrieve tags from current NetBox instance + self.get_instance_tags() + dependencies = { - "tags": [ - { - "name": "Orphaned", - "slug": "orphaned", - "color": "607d8b", - "description": "The source system which has previously" - "provided the object no longer " - "states it exists.{}".format( - " An object with the 'Orphaned' tag will " - "remain in this state until it ages out " - "and is automatically removed." - ) if settings.NB_PRUNE_ENABLED else "" - }, - { - "name": self.vc_tag, - "slug": format_slug(self.vc_tag), - "description": "Objects synced from vCenter host " - "{}. Be careful not to modify the name or " - "slug.".format(self.vc_tag) - }, - { - "name": "vCenter", - "slug": "vcenter", - "description": "Created and used by vCenter NetBox sync." - }], "manufacturers": [ {"name": "VMware", "slug": "vmware"}, ], @@ -1586,12 +1656,24 @@ def verify_dependencies(self): log.info("Verifying all prerequisite objects exist in NetBox.") for dep_type in dependencies: log.debug( - "Checking NetBox has necessary %s objects.", dep_type[:-1] + "Checking that NetBox has necessary %s objects.", dep_type[:-1] ) for dep in dependencies[dep_type]: self.obj_exists(nb_obj_type=dep_type, vc_data=dep) log.info("Finished verifying prerequisites.") + def get_instance_tags(self): + """ + retrieve all tags from this NetBox instance + """ + + log.debug("Retrieve all tags from this NetBox instance") + + req = self.request( + req_type="get", nb_obj_type="tags") + + self.instance_tags = req.get("results") + def remove_all(self): """ Searches NetBox for all synced objects and then removes them. diff --git a/templates/netbox.py b/templates/netbox.py index 2350769..0f8a71c 100644 --- a/templates/netbox.py +++ b/templates/netbox.py @@ -48,6 +48,9 @@ def truncate(text="", max_len=50): :return: Text in :param text: truncated to :param max_len: if necessary :rtype: str """ + if text is None: + return None + return text if len(text) < max_len else text[:max_len] @@ -760,3 +763,34 @@ def vrf(self, name, rd=None, tenant=None, enforce_unique=True, "tags": tags } return remove_empty_fields(obj) + + def tag(self, name, slug=None, id=None, url=None, color=None, + description=None, tagged_items=None): + """ + Template for NetBox TAGs at /extras/tags/ + + :param name: Name of the TAG + :type name: str + :param slug: Unique slug for tag + :type slug: str, optional + :param id: Unique id for the TAG + :type id: int, optional + :param url: Unique url of tag in NetBox instance + :type url: str, optional + :param color: hexadecimal web color representation + :type color: str, optional + :param description: Description of the TAG, max length 200 + :type description: str, optional + :param tagged_items: number of items in NetBox instance with this tag assigned + :type tagged_items: int, optional + + """ + return remove_empty_fields({ + "id": id, + "name": truncate(name, max_len=100), + "slug": slug if slug else format_slug(name, max_len=100), + "url": url, + "description": truncate(description, max_len=200), + "color": truncate(color, max_len=6), + "tagged_items": tagged_items + }) From 9447109a12e18703954cd843c7bc2a6adb936285 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Wed, 23 Sep 2020 14:15:45 +0200 Subject: [PATCH 10/10] fixes assignment of interfaces to ip addresses and interfaces to devices/vms * after NetBox changed the API 2.9 this assignment process was broken --- run.py | 157 +++++++++++++++++++++++++++++++++++--------- templates/netbox.py | 33 ++++------ 2 files changed, 138 insertions(+), 52 deletions(-) diff --git a/run.py b/run.py index 08c1554..4b481ec 100644 --- a/run.py +++ b/run.py @@ -440,7 +440,7 @@ def get_objects(self, vc_obj_type): results = {} # Setup use of NetBox templates nbt = Templates(api_version=self.nb_api_version) - # Initalize keys expected to be returned + # Initialize keys expected to be returned for nb_obj_type in obj_type_map[vc_obj_type]: results.setdefault(nb_obj_type, []) # Create vCenter view for object collection @@ -641,7 +641,7 @@ def get_objects(self, vc_obj_type): ip_addr, vnic.spec.ip.subnetMask ), device=truncate(obj_name, max_len=64), - interface=nic_name, + assigned_object=nic_name, tags=self.tags, )) elif vc_obj_type == "virtual_machines": @@ -697,7 +697,6 @@ def get_objects(self, vc_obj_type): results["virtual_interfaces"].append( nbt.vm_interface( virtual_machine=obj_name, - itype=0, name=nic_name, mac_address=nic.macAddress, enabled=nic.connected, @@ -717,7 +716,7 @@ def get_objects(self, vc_obj_type): nbt.ip_address( address=ip_addr, virtual_machine=obj_name, - interface=nic_name, + assigned_object=nic_name, tags=self.tags )) except AttributeError as err: @@ -742,6 +741,8 @@ class NetBoxHandler: """ instance_tags = None + instance_interfaces = {} + instance_virtual_interfaces = {} def __init__(self, vc_conn): self.nb_api_url = "http{}://{}{}/api/".format( @@ -1009,6 +1010,9 @@ def single_request(self, req_type, nb_obj_type, data=None, query=None, nb_id=Non "created" if req.status_code == 201 else "deleted", nb_obj_type, ) + # return patched data + if req.status_code == 201: + result = req.json() elif req.status_code == 400: if req_type == "post": log.warning( @@ -1068,6 +1072,8 @@ def update_tag_data(self, tag_list): """ nbt = Templates(api_version=self.nb_api_version) + tagdata_updated = False + for tag in tag_list: if isinstance(tag, dict): @@ -1087,8 +1093,11 @@ def update_tag_data(self, tag_list): data=nbt.tag(name=tag) ) + tagdata_updated = True + # update tag information - self.get_instance_tags() + if tagdata_updated: + self.get_instance_tags() return [{"name": d } for d in tag_list] @@ -1134,6 +1143,22 @@ def obj_exists(self, nb_obj_type, vc_data): query=query ) + # add interface id and type to ip_address object + if nb_obj_type == "ip_addresses": + nic_name = vc_data["assigned_object"]["name"] + if vc_data["assigned_object"].get("virtual_machine"): + name = vc_data["assigned_object"]["virtual_machine"]["name"] + int_data = self.instance_virtual_interfaces["{}/{}".format(name, nic_name)] + int_type = 'virtualization.vminterface' + if vc_data["assigned_object"].get("device"): + name = vc_data["assigned_object"]["device"]["name"] + int_data = self.instance_interfaces["{}/{}".format(name, nic_name)] + int_type = 'dcim.interface' + + if int_data is not None: + vc_data["assigned_object_id"] = int_data.get("id") + vc_data["assigned_object_type"] = int_type + # A single matching object is found so we compare its values to the new # object if req["count"] == 1: @@ -1143,6 +1168,9 @@ def obj_exists(self, nb_obj_type, vc_data): ) nb_data = req["results"][0] + if "interfaces" in nb_obj_type: + self.update_interface_data(nb_obj_type, nb_data) + # reduce NetBox tags to names only if nb_data.get("tags"): nb_data["tags"] = [{"name": d["name"]} for d in nb_data["tags"]] @@ -1172,6 +1200,41 @@ def obj_exists(self, nb_obj_type, vc_data): nb_obj_type, vc_data[query_key] ) else: + # prevent reassignment of ip_addresses which have been assigned + # to a different virtual_machine in NetBox + if nb_obj_type == "ip_addresses" and nb_data.get("assigned_object") is not None: + + vc_object_name = None + if vc_data.get("assigned_object") is not None: + if vc_data.get("assigned_object").get("device"): + vc_object_name = vc_data["assigned_object"]["device"]["name"] + if vc_data.get("assigned_object").get("virtual_machine"): + vc_object_name = vc_data["assigned_object"]["virtual_machine"]["name"] + + if nb_data["assigned_object"].get("virtual_machine") is not None: + if nb_data["assigned_object"]["virtual_machine"]["name"] != vc_object_name: + + log.warning( + "NetBox %s object '%s' for '%s' is already " + "assigned to a different virtual_machine '%s'. " + "Skipping for safety.", + nb_obj_type, vc_data[query_key], vc_object_name, + nb_data["assigned_object"]["virtual_machine"]["name"] + ) + return + + if nb_data["assigned_object"].get("device") is not None: + if nb_data["assigned_object"]["device"]["name"] != vc_object_name: + + log.warning( + "NetBox %s object '%s' for '%s' is already " + "assigned to a different device '%s'. " + "Skipping for safety.", + nb_obj_type, vc_data[query_key], vc_object_name, + nb_data["assigned_object"]["device"]["name"] + ) + return + log.info( "NetBox %s object '%s' do not match current values.", nb_obj_type, vc_data[query_key] @@ -1192,10 +1255,13 @@ def obj_exists(self, nb_obj_type, vc_data): "Removed site from %s object before sending update " "to NetBox.", vc_data[query_key] ) - self.request( + respsone = self.request( req_type="patch", nb_obj_type=nb_obj_type, data=vc_data, nb_id=nb_data["id"] ) + if "interfaces" in nb_obj_type: + self.update_interface_data(nb_obj_type, respsone) + elif req["count"] > 1: log.warning( "Search for NetBox %s object '%s' returned %s results but " @@ -1213,10 +1279,13 @@ def obj_exists(self, nb_obj_type, vc_data): nb_obj_type, vc_data[query_key], ) - self.request( + respsone = self.request( req_type="post", nb_obj_type=nb_obj_type, data=vc_data ) + if "interfaces" in nb_obj_type: + self.update_interface_data(nb_obj_type, respsone) + def set_primary_ips(self): """Sets the Primary IP of vCenter hosts and Virtual Machines.""" for nb_obj_type in ("devices", "virtual_machines"): @@ -1295,7 +1364,7 @@ def set_dns_names(self): req_type="get", nb_obj_type="ip_addresses", query="?tag={}".format(format_slug(self.vc_tag)) )["results"] - log.info("Collected %s NetBox IP address objects.", ip_objs) + log.info("Collected %s NetBox IP address objects.", len(ip_objs)) # We take the IP address objects and make a map of relevant details to # compare against and use later nb_objs = {} @@ -1308,7 +1377,7 @@ def set_dns_names(self): ips = [ip["address"].split("/")[0] for ip in ip_objs] ptrs = queue_dns_lookups(ips) # Having collected the IP address objects from NetBox already we can - # avoid individual checks for updates by comparing the objects and ptrs + # avoid individual checks for updates by comparing the objects and PTRs log.info("Comparing latest PTR records against existing NetBox data.") for ip, ptr in ptrs: if ptr != nb_objs[ip]["dns_name"]: @@ -1421,33 +1490,34 @@ def prune_objects(self, vc_objects, vc_obj_type): )["results"] # Certain vCenter object types map to multiple NetBox types. We # define the relationships to compare against for these situations. - if vc_obj_type == "hosts" and nb_obj_type == "interfaces": - nb_objects = [ - obj for obj in nb_objects - if obj["device"] is not None + if vc_obj_type == "hosts": + if nb_obj_type == "interfaces": + nb_objects = [ + obj for obj in nb_objects + if obj["device"] is not None ] - elif vc_obj_type == "hosts" and nb_obj_type == "ip_addresses": - nb_objects = [ - obj for obj in nb_objects - if obj["interface"]["device"] is not None + elif nb_obj_type == "ip_addresses": +# pprint(nb_objects) + nb_objects = [ + obj for obj in nb_objects + if obj.get("assigned_object") is not None and obj.get("assigned_object").get("device") is not None ] # Issue 33: As of NetBox v2.6.11 it is not possible to filter # virtual interfaces by tag. Therefore we filter post collection. - elif vc_obj_type == "virtual_machines" and \ - nb_obj_type == "virtual_interfaces": - nb_objects = [ - obj for obj in nb_objects - if self.vc_tag in obj["tags"] + elif vc_obj_type == "virtual_machines": + if nb_obj_type == "virtual_interfaces": + nb_objects = [ + obj for obj in nb_objects + if self.vc_tag in obj["tags"] ] - log.debug( - "Found %s virtual interfaces with tag '%s'.", - len(nb_objects), self.vc_tag + log.debug( + "Found %s virtual interfaces with tag '%s'.", + len(nb_objects), self.vc_tag ) - elif vc_obj_type == "virtual_machines" and \ - nb_obj_type == "ip_addresses": - nb_objects = [ - obj for obj in nb_objects - if obj["interface"]["virtual_machine"] is not None + elif nb_obj_type == "ip_addresses": + nb_objects = [ + obj for obj in nb_objects + if obj.get("assigned_object") is not None and obj.get("assigned_object").get("virtual_machine") is not None ] # From the vCenter objects provided collect only the names/models of # each object from the current type we're comparing against @@ -1471,7 +1541,7 @@ def prune_objects(self, vc_objects, vc_obj_type): "Processing orphaned NetBox %s '%s' object.", nb_obj_type, orphan[query_key] ) - if "Orphaned" not in orphan["tags"]: + if "Orphaned" not in [d.get("name") for d in orphan["tags"]]: log.info( "No tag found. Adding 'Orphaned' tag to %s '%s' " "object.", @@ -1479,7 +1549,9 @@ def prune_objects(self, vc_objects, vc_obj_type): ) log.debug("Merging existing tags with `Orphaned`.") tags = { - "tags": list(set(orphan["tags"] + ["Orphaned"])) + "tags": list( + {'name':x['name']} for x in orphan["tags"] + ) + [{"name": "Orphaned"}] } self.request( req_type="patch", nb_obj_type=nb_obj_type, @@ -1674,6 +1746,27 @@ def get_instance_tags(self): self.instance_tags = req.get("results") + def update_interface_data(self, interface_type, interface_data): + """ + Add NetBox interface to local lookup list so we are able to + assign a interface to a NetBox ip_addresses ressource + """ + + object_name = None + if interface_type == "virtual_interfaces": + object_name = interface_data.get("virtual_machine").get("name") + if interface_type == "interfaces": + object_name = interface_data.get("device").get("name") + + interface_name = interface_data.get("name") + + if object_name is not None and interface_name is not None: + key = "{}/{}".format(object_name, interface_name) + if interface_type == "virtual_interfaces": + self.instance_virtual_interfaces[key] = interface_data + if interface_type == "interfaces": + self.instance_interfaces[key] = interface_data + def remove_all(self): """ Searches NetBox for all synced objects and then removes them. diff --git a/templates/netbox.py b/templates/netbox.py index 0f8a71c..838a291 100644 --- a/templates/netbox.py +++ b/templates/netbox.py @@ -382,7 +382,7 @@ def device_type(self, manufacturer, model, slug=None, part_number=None, return remove_empty_fields(obj) def ip_address(self, address, description=None, device=None, dns_name=None, - interface=None, status=1, tags=None, tenant=None, + assigned_object=None, status=1, tags=None, tenant=None, virtual_machine=None, vrf=None): """ Template for NetBox IP addresses at /ipam/ip-addresses/ @@ -391,12 +391,12 @@ def ip_address(self, address, description=None, device=None, dns_name=None, :type address: str :param description: A description of the IP address purpose :type description: str, optional - :param device: The device which the IP and its interface are attached to + :param device: The device which the IP and its assigned_object are attached to :type device: str, optional :param dns_name: FQDN pointed to the IP address :type dns_name: str, optional - :param interface: Name of the parent interface IP is configured on - :type interface: str, optional + :param assigned_object: Name of the parent assigned_object IP is configured on + :type assigned_object: str, optional :param status: `1` if active, `0` if deprecated :type status: int :param tags: Tags to apply to the object @@ -427,15 +427,15 @@ def ip_address(self, address, description=None, device=None, dns_name=None, "tenant": tenant, "vrf": vrf } - if interface and bool(device or virtual_machine): - obj["interface"] = {"name": interface} + if assigned_object and bool(device or virtual_machine): + obj["assigned_object"] = {"name": assigned_object} if device: - obj["interface"] = { - **obj["interface"], **{"device": {"name": device}} + obj["assigned_object"] = { + **obj["assigned_object"], **{"device": {"name": device}} } elif virtual_machine: - obj["interface"] = { - **obj["interface"], + obj["assigned_object"] = { + **obj["assigned_object"], **{"virtual_machine": { "name": truncate(virtual_machine, max_len=64) }} @@ -686,7 +686,7 @@ def vlan(self, vid, name, site=None, group=None, tenant=None, status=None, } return remove_empty_fields(obj) - def vm_interface(self, virtual_machine, name, itype=0, enabled=None, + def vm_interface(self, virtual_machine, name, enabled=None, mtu=None, mac_address=None, description=None, mode=None, untagged_vlan=None, tagged_vlans=None, tags=None): """ @@ -696,8 +696,6 @@ def vm_interface(self, virtual_machine, name, itype=0, enabled=None, :type virtual_machine: str :param name: Name of the physical interface :type name: str - :param itype: Type of interface `0` if Virtual else `32767` for Other - :type itype: str, optional :param enabled: `True` if the interface is up else `False` :type enabled: bool,optional :param mtu: The configured MTU for the interface @@ -705,9 +703,9 @@ def vm_interface(self, virtual_machine, name, itype=0, enabled=None, :param mac_address: The MAC address of the interface :type mac_address: str, optional :param description: Description for the interface - :itype description: str, optional + :type description: str, optional :param mode: `100` if access, `200` if tagged, or `300 if` tagged for all vlans - :itype mode: int, optional + :type mode: int, optional :param untagged_vlan: NetBox VLAN object id of untagged vlan :type untagged_vlan: int, optional :param tagged_vlans: List of NetBox VLAN object ids for tagged VLANs @@ -718,11 +716,6 @@ def vm_interface(self, virtual_machine, name, itype=0, enabled=None, obj = { "virtual_machine": {"name": truncate(virtual_machine, max_len=64)}, "name": name, - "itype": self._version_dependent( - nb_obj_type="interfaces", - key="type", - value=itype - ), "enabled": enabled, "mtu": mtu, "mac_address": mac_address.upper() if mac_address else None,