From 92711a92ed0085d930ed464bb8c5c8faeae1972f Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Thu, 20 Mar 2025 22:41:53 +0100 Subject: [PATCH 01/27] enhances NetBox version detection #446 --- module/netbox/connection.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/module/netbox/connection.py b/module/netbox/connection.py index 5c86d26f..d3a0dd56 100644 --- a/module/netbox/connection.py +++ b/module/netbox/connection.py @@ -196,15 +196,29 @@ def get_api_version(self): str: NetBox API version """ response = None + result = None + try: response = self.session.get( - self.url, + f"{self.url}/status", timeout=self.settings.timeout, verify=self.settings.validate_tls_certs) except Exception as e: do_error_exit(f"NetBox connection: {e}") - result = str(response.headers.get("API-Version")) + # noinspection PyBroadException + try: + result = grab(response.json(), "netbox-version").split("-")[0] + except Exception: + pass + + if not isinstance(result, str): + result = str(response.headers.get("API-Version")) + + try: + version.parse(result) + except Exception as e: + do_error_exit(f"Unable to parse NetBox version '{result}': {e}") log.info(f"Successfully connected to NetBox '{self.settings.host_fqdn}'") log.debug(f"Detected NetBox API version: {result}") From 720a3ba47a78ca1144e379d13d66a9642808ef86 Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Fri, 21 Mar 2025 07:39:55 +0100 Subject: [PATCH 02/27] fixes issue with retrieving platform name for newer guest tools #448 --- module/sources/vmware/connection.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 9ab89744..1b3be800 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -2180,10 +2180,13 @@ def add_virtual_machine(self, obj): platform = get_string_or_none(grab(obj, "guest.guestFullName", fallback=platform)) # extract prettyName from extraConfig exposed by guest tools - extra_config = [x.value for x in grab(obj, "config.extraConfig", fallback=[]) - if x.key == "guestOS.detailed.data"] - if len(extra_config) > 0: - pretty_name = [x for x in quoted_split(extra_config[0].replace("' ", "', ")) if x.startswith("prettyName")] + extra_config = {x.key: x.value for x in grab(obj, "config.extraConfig", fallback=[]) + if x.key in ["guestOS.detailed.data", "guestInfo.detailed.data"]} + + # first try 'guestInfo.detailed.data' and then 'guestOS.detailed.data' + detailed_data = extra_config.get("guestInfo.detailed.data") or extra_config.get("guestOS.detailed.data") + if isinstance(detailed_data, str): + pretty_name = [x for x in quoted_split(detailed_data.replace("' ", "', ")) if x.startswith("prettyName")] if len(pretty_name) > 0: platform = pretty_name[0].replace("prettyName='","") From edf0495e64fc143fe6d43edb4ecd6b1266c97da1 Mon Sep 17 00:00:00 2001 From: Littlericket <1573629+Littlericket@users.noreply.github.com> Date: Fri, 21 Mar 2025 15:33:51 +0100 Subject: [PATCH 03/27] fix: handle object and multi-object custom fields correctly --- module/netbox/object_classes.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 96df5fd9..9d250935 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -683,6 +683,29 @@ def update(self, data=None, read_from_netbox=False, source=None): if self.data_model.get(key) == NBCustomField: if current_value is None: current_value = dict() + + # Fix for object/multi-object custom fields + # When patching, we only need the IDs, not the full object representation + new_value_copy = new_value.copy() + for field_name, field_value in new_value_copy.items(): + # Check for custom field type + custom_field = self.inventory.get_by_data(NBCustomField, data={"name": field_name}) + if custom_field is not None: + field_type = grab(custom_field, "data.type") + + # Handle object type custom fields - need only ID + if field_type == "object" and isinstance(field_value, dict) and field_value.get('id') is not None: + new_value[field_name] = field_value.get('id') + + # Handle multi-object type custom fields - need list of IDs + elif field_type == "multi-object" and isinstance(field_value, list): + ids = [] + for item in field_value: + if isinstance(item, dict) and item.get('id') is not None: + ids.append(item.get('id')) + if ids: + new_value[field_name] = ids + new_value = {**current_value, **new_value} new_value_str = str(new_value) elif isinstance(new_value, (NetBoxObject, NBObjectList)): @@ -1288,7 +1311,7 @@ def __init__(self, *args, **kwargs): "object_types": list, # field name (object_types) for NetBox < 4.0.0 "content_types": list, - "type": ["text", "longtext", "integer", "boolean", "date", "url", "json", "select", "multiselect"], + "type": ["text", "longtext", "integer", "boolean", "date", "url", "json", "select", "multiselect", "object", "multi-object"], "name": 50, "label": 50, "description": 200, From 7c14187ef2a41c1cde9b00b9f08b392af2d0925f Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Fri, 9 May 2025 13:05:06 +0200 Subject: [PATCH 04/27] fixes issue with type hints for python <3.10.0 setups #456 --- module/sources/common/source_base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index fc81a160..0e95e5cc 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -8,6 +8,7 @@ # repository or visit: . import re +from typing import Union,Optional from ipaddress import ip_interface, ip_address, IPv6Address, IPv4Address, IPv6Network, IPv4Network from packaging import version @@ -179,7 +180,7 @@ def map_object_interfaces_to_current_interfaces(self, device_vm_object, interfac return return_data - def return_longest_matching_prefix_for_ip(self, ip_to_match=None, site_name=None) -> NBPrefix|None: + def return_longest_matching_prefix_for_ip(self, ip_to_match=None, site_name=None) -> Optional[NBPrefix]: """ This is a lazy approach to find the longest matching prefix to an IP address. If site_name is set only IP prefixes from that site are matched. @@ -716,7 +717,7 @@ def patch_data(object_to_patch, data, overwrite=False): return data_to_update - def add_vlan_group(self, vlan_data, vlan_site, vlan_cluster) -> NBVLAN | dict: + def add_vlan_group(self, vlan_data, vlan_site, vlan_cluster) -> Union[NBVLAN ,dict]: """ This function will try to find a matching VLAN group according to the settings. Name matching will take precedence over ID matching. First match wins. From cee3425e0c628f9b6894ba65b649ea81b998de0f Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Tue, 17 Jun 2025 15:06:26 +0200 Subject: [PATCH 05/27] adds distroVersion to VM linux platform #448 --- module/sources/vmware/connection.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 1b3be800..497f04f9 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -2186,9 +2186,15 @@ def add_virtual_machine(self, obj): # first try 'guestInfo.detailed.data' and then 'guestOS.detailed.data' detailed_data = extra_config.get("guestInfo.detailed.data") or extra_config.get("guestOS.detailed.data") if isinstance(detailed_data, str): - pretty_name = [x for x in quoted_split(detailed_data.replace("' ", "', ")) if x.startswith("prettyName")] - if len(pretty_name) > 0: - platform = pretty_name[0].replace("prettyName='","") + detailed_data_dict = dict() + for detailed_data_item in quoted_split(detailed_data.replace("' ", "', ")): + detailed_data_key, detailed_data_value = detailed_data_item.split("=") + detailed_data_dict[detailed_data_key] = detailed_data_value.strip("'") + if len(detailed_data_dict.get("prettyName","")) > 0: + platform = detailed_data_dict.get("prettyName") + if detailed_data_dict.get("prettyName").lower() == "linux" and \ + detailed_data_dict.get("distroVersion") not in platform: + platform = f'{platform} {detailed_data_dict.get("distroVersion")}' if platform is not None: platform = self.get_object_relation(platform, "vm_platform_relation", fallback=platform) From 15484950586eacf71e44b9bdbf7f952a081dcd0a Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Wed, 18 Jun 2025 12:06:17 +0200 Subject: [PATCH 06/27] fixes issue with parsing of vm guest data #448 --- module/sources/vmware/connection.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 497f04f9..02a1765e 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -2188,11 +2188,14 @@ def add_virtual_machine(self, obj): if isinstance(detailed_data, str): detailed_data_dict = dict() for detailed_data_item in quoted_split(detailed_data.replace("' ", "', ")): + if "=" not in detailed_data_item: + continue + detailed_data_key, detailed_data_value = detailed_data_item.split("=") detailed_data_dict[detailed_data_key] = detailed_data_value.strip("'") if len(detailed_data_dict.get("prettyName","")) > 0: platform = detailed_data_dict.get("prettyName") - if detailed_data_dict.get("prettyName").lower() == "linux" and \ + if detailed_data_dict.get("familyName", "").lower() == "linux" and \ detailed_data_dict.get("distroVersion") not in platform: platform = f'{platform} {detailed_data_dict.get("distroVersion")}' From 328c70b9f1603e49f4682c3c5666094841db0a4f Mon Sep 17 00:00:00 2001 From: joachimBurket Date: Thu, 3 Jul 2025 17:41:43 +0200 Subject: [PATCH 07/27] only set the Cluster scope_type if the site_name is not None --- module/sources/vmware/connection.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 9ab89744..4d17d5e6 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -1388,8 +1388,9 @@ def add_cluster(self, obj): } if version.parse(self.inventory.netbox_api_version) >= version.parse("4.2.0"): - data["scope_id"] = {"name": site_name} - data["scope_type"] = "dcim.site" + if site_name is not None: + data["scope_id"] = {"name": site_name} + data["scope_type"] = "dcim.site" else: data["site"] = {"name": site_name} From 1320d8105cb8b4eb86391bbac33466200033542f Mon Sep 17 00:00:00 2001 From: Ricardo Bartels Date: Wed, 30 Jul 2025 09:27:08 +0200 Subject: [PATCH 08/27] adds note to readme about sunsetting this project #474 --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 0430a672..b944ac26 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # NetBox-Sync +> [!CAUTION] +> **Maintainer wanted - sunsetting this repository by 31.10.2025 [#474](https://github.com/bb-Ricardo/netbox-sync/issues/474)** + This is a tool to sync data from different sources to a NetBox instance. Available source types: From b080e4447d2393b7e122bebcb9ef9a202084b44d Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 25 Aug 2025 12:17:22 +1000 Subject: [PATCH 09/27] stops ip addresses assigned to FHRP groups from being reassigned --- module/netbox/object_classes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 9d250935..f0323833 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2152,6 +2152,10 @@ def update(self, data=None, read_from_netbox=False, source=None): object_type = data.get("assigned_object_type") assigned_object = data.get("assigned_object_id") + if object_type == "ipam.fhrpgroup": + log.info("IP address assigned to FHRP group. Skipping.") + return + # used to track changes in object primary IP assignments previous_ip_device_vm = None is_primary_ipv4_of_previous_device = False From 049655a8001f72eb223bcfe850a5e7a8ff413593 Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 25 Aug 2025 13:24:10 +1000 Subject: [PATCH 10/27] worded the log better --- module/netbox/object_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index f0323833..194b6256 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2153,7 +2153,7 @@ def update(self, data=None, read_from_netbox=False, source=None): assigned_object = data.get("assigned_object_id") if object_type == "ipam.fhrpgroup": - log.info("IP address assigned to FHRP group. Skipping.") + log.info(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") return # used to track changes in object primary IP assignments From 5f16691744c31eccafff9df9731b6b1bdb44093b Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 25 Aug 2025 15:42:31 +1000 Subject: [PATCH 11/27] added skipping_fhrp_group_ips config option to enable or disable fhrp group ip address overriding --- module/netbox/object_classes.py | 14 ++++++++++---- module/sources/check_redfish/config.py | 6 ++++++ module/sources/vmware/config.py | 5 +++++ module/sources/vmware/connection.py | 6 ++++++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 194b6256..dfa49f56 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2148,12 +2148,18 @@ def resolve_relations(self): super().resolve_relations() def update(self, data=None, read_from_netbox=False, source=None): - object_type = data.get("assigned_object_type") assigned_object = data.get("assigned_object_id") - - if object_type == "ipam.fhrpgroup": - log.info(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") + + # Skip IP assignments when the IP is assigned to FHRP groups when config option + # skipping_fhrp_group_ips is set to True + if source is not None: + config_relation = source.get_object_relation(assigned_object, "skipping_fhrp_group_ips") + if config_relation == True and object_type == "ipam.fhrpgroup": + log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") + return + elif object_type == "ipam.fhrpgroup": + log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") return # used to track changes in object primary IP assignments diff --git a/module/sources/check_redfish/config.py b/module/sources/check_redfish/config.py index 2c21fff9..2446671a 100644 --- a/module/sources/check_redfish/config.py +++ b/module/sources/check_redfish/config.py @@ -70,6 +70,12 @@ def __init__(self): via check_redfish if False only data which is not preset in NetBox will be added""", default_value=False), + ConfigOption("skipping_fhrp_group_ips", + bool, + description="""define if an IP address assigned to a FHRP group (like HSRP, VRRP, GLBP) will be skipped. + If True this IP address will be skipped and not synced to NetBox to prevent incorrect syncing.""", + default_value=False), + ConfigOption(**config_option_ip_tenant_inheritance_order_definition), ] diff --git a/module/sources/vmware/config.py b/module/sources/vmware/config.py index 7e448a28..14acdec8 100644 --- a/module/sources/vmware/config.py +++ b/module/sources/vmware/config.py @@ -258,6 +258,11 @@ def __init__(self): description="""If the VMware Site Recovery Manager is used to can skip syncing placeholder/replicated VMs from fail-over site to NetBox.""", default_value=False), + ConfigOption("skipping_fhrp_group_ips", + bool, + description="""If an IP address is assigned to a FHRP group (like HSRP, VRRP, GLBP) + then this IP address will be skipped and not synced to NetBox to prevent incorrect syncing.""", + default_value=False), ConfigOption("strip_host_domain_name", bool, description="strip domain part from host name before syncing device to NetBox", diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 742f54ca..d275742c 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -914,6 +914,12 @@ def get_object_relation(self, name, relation, fallback=None): """ resolved_list = list() + relation_data = grab(self.settings, relation, fallback=fallback) + + if isinstance(relation_data, bool): + log.debug(f"Object relation '{relation}' is boolean '{relation_data}'. Returning.") + return relation_data + for single_relation in grab(self.settings, relation, fallback=list()): object_regex = single_relation.get("object_regex") match_found = False From 960680c47ead2cd9292c5e9fb69fccfacee37eae Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 25 Aug 2025 16:23:45 +1000 Subject: [PATCH 12/27] limited skipping_fhrp_group_ips to vmware since I can't test redfish presently --- module/netbox/object_classes.py | 11 ++++++----- module/sources/vmware/connection.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index dfa49f56..3f0397b1 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2154,12 +2154,13 @@ def update(self, data=None, read_from_netbox=False, source=None): # Skip IP assignments when the IP is assigned to FHRP groups when config option # skipping_fhrp_group_ips is set to True if source is not None: - config_relation = source.get_object_relation(assigned_object, "skipping_fhrp_group_ips") - if config_relation == True and object_type == "ipam.fhrpgroup": - log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") - return + if source.source_type == "vmware": + config_relation = source.get_object_relation(assigned_object, "skipping_fhrp_group_ips") + if config_relation == True and object_type == "ipam.fhrpgroup": + log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") + return elif object_type == "ipam.fhrpgroup": - log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") + log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. It was manually created. Skipping.") return # used to track changes in object primary IP assignments diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index d275742c..542f08a2 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -917,7 +917,7 @@ def get_object_relation(self, name, relation, fallback=None): relation_data = grab(self.settings, relation, fallback=fallback) if isinstance(relation_data, bool): - log.debug(f"Object relation '{relation}' is boolean '{relation_data}'. Returning.") + log.debug(f"Object relation '{relation}' is boolean, set '{relation_data}'.") return relation_data for single_relation in grab(self.settings, relation, fallback=list()): From dae2911659fca4df8967222dbe6b9b16da42ac31 Mon Sep 17 00:00:00 2001 From: Noah Date: Mon, 25 Aug 2025 16:27:09 +1000 Subject: [PATCH 13/27] tweaked comment --- module/netbox/object_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 3f0397b1..455a833e 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2152,7 +2152,7 @@ def update(self, data=None, read_from_netbox=False, source=None): assigned_object = data.get("assigned_object_id") # Skip IP assignments when the IP is assigned to FHRP groups when config option - # skipping_fhrp_group_ips is set to True + # skipping_fhrp_group_ips is set to True, or if the IP is manually assigned to an FHRP group (no source) if source is not None: if source.source_type == "vmware": config_relation = source.get_object_relation(assigned_object, "skipping_fhrp_group_ips") From b61a34bbc979e98cfa29d1c615521a6829f60164 Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 26 Aug 2025 09:54:04 +1000 Subject: [PATCH 14/27] limited skip_fhrp_group_ips to vmware since I can't test redfish presently --- module/netbox/object_classes.py | 13 +++++++------ module/sources/check_redfish/config.py | 2 +- module/sources/vmware/config.py | 2 +- module/sources/vmware/connection.py | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index dfa49f56..0b954a61 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2152,14 +2152,15 @@ def update(self, data=None, read_from_netbox=False, source=None): assigned_object = data.get("assigned_object_id") # Skip IP assignments when the IP is assigned to FHRP groups when config option - # skipping_fhrp_group_ips is set to True + # skip_fhrp_group_ips is set to True, or if the IP is manually assigned to an FHRP group (no source) if source is not None: - config_relation = source.get_object_relation(assigned_object, "skipping_fhrp_group_ips") - if config_relation == True and object_type == "ipam.fhrpgroup": - log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") - return + if source.source_type == "vmware": + config_relation = source.get_object_relation(assigned_object, "skip_fhrp_group_ips") + if config_relation == True and object_type == "ipam.fhrpgroup": + log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") + return elif object_type == "ipam.fhrpgroup": - log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") + log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. It was manually created. Skipping.") return # used to track changes in object primary IP assignments diff --git a/module/sources/check_redfish/config.py b/module/sources/check_redfish/config.py index 2446671a..a4f99be3 100644 --- a/module/sources/check_redfish/config.py +++ b/module/sources/check_redfish/config.py @@ -70,7 +70,7 @@ def __init__(self): via check_redfish if False only data which is not preset in NetBox will be added""", default_value=False), - ConfigOption("skipping_fhrp_group_ips", + ConfigOption("skip_fhrp_group_ips", bool, description="""define if an IP address assigned to a FHRP group (like HSRP, VRRP, GLBP) will be skipped. If True this IP address will be skipped and not synced to NetBox to prevent incorrect syncing.""", diff --git a/module/sources/vmware/config.py b/module/sources/vmware/config.py index 14acdec8..3e5c731d 100644 --- a/module/sources/vmware/config.py +++ b/module/sources/vmware/config.py @@ -258,7 +258,7 @@ def __init__(self): description="""If the VMware Site Recovery Manager is used to can skip syncing placeholder/replicated VMs from fail-over site to NetBox.""", default_value=False), - ConfigOption("skipping_fhrp_group_ips", + ConfigOption("skip_fhrp_group_ips", bool, description="""If an IP address is assigned to a FHRP group (like HSRP, VRRP, GLBP) then this IP address will be skipped and not synced to NetBox to prevent incorrect syncing.""", diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index d275742c..542f08a2 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -917,7 +917,7 @@ def get_object_relation(self, name, relation, fallback=None): relation_data = grab(self.settings, relation, fallback=fallback) if isinstance(relation_data, bool): - log.debug(f"Object relation '{relation}' is boolean '{relation_data}'. Returning.") + log.debug(f"Object relation '{relation}' is boolean, set '{relation_data}'.") return relation_data for single_relation in grab(self.settings, relation, fallback=list()): From 7e0fde97d45bafa41d3f222ecbff479fcbc39cc1 Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 26 Aug 2025 13:09:15 +1000 Subject: [PATCH 15/27] modified: module/sources/vmware/connection.py --- module/sources/vmware/connection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 542f08a2..a493d37d 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -519,6 +519,7 @@ def get_object_based_on_macs(self, object_type, mac_list=None): Returns ------- (NBDevice, NBVM, None): object instance of found device, otherwise None + comment line """ object_to_return = None From ee80b802d9d88f98ce3f5540dede5448d76e3974 Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 26 Aug 2025 13:12:13 +1000 Subject: [PATCH 16/27] modified: module/sources/vmware/connection.py --- module/sources/vmware/connection.py | 1 - 1 file changed, 1 deletion(-) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index a493d37d..542f08a2 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -519,7 +519,6 @@ def get_object_based_on_macs(self, object_type, mac_list=None): Returns ------- (NBDevice, NBVM, None): object instance of found device, otherwise None - comment line """ object_to_return = None From 0be06699759a27faa9d1f10c4b9a4b7f2fe621c9 Mon Sep 17 00:00:00 2001 From: Noah Date: Tue, 26 Aug 2025 16:53:44 +1000 Subject: [PATCH 17/27] changed the NBIPAddress data model entry 'assigned_object_type' to have set valid types --- module/netbox/object_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 0b954a61..999c8527 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2131,7 +2131,7 @@ def __init__(self, *args, **kwargs): ] self.data_model = { "address": str, - "assigned_object_type": self.mapping.scopes_object_types(self.scopes), + "assigned_object_type": ("ipam.fhrpgroup, dcim.interface, virtualization.vminterface"), "assigned_object_id": self.scopes, "description": 200, "role": ["loopback", "secondary", "anycast", "vip", "vrrp", "hsrp", "glbp", "carp"], From 2e3fb455f3a4c89508738281125a614bff4191fe Mon Sep 17 00:00:00 2001 From: Noah Date: Wed, 27 Aug 2025 09:34:14 +1000 Subject: [PATCH 18/27] modified: module/netbox/object_classes.py --- module/netbox/object_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 999c8527..0b954a61 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2131,7 +2131,7 @@ def __init__(self, *args, **kwargs): ] self.data_model = { "address": str, - "assigned_object_type": ("ipam.fhrpgroup, dcim.interface, virtualization.vminterface"), + "assigned_object_type": self.mapping.scopes_object_types(self.scopes), "assigned_object_id": self.scopes, "description": 200, "role": ["loopback", "secondary", "anycast", "vip", "vrrp", "hsrp", "glbp", "carp"], From 6ffa661705e39c5590bc787ea4a39a30443d4fe9 Mon Sep 17 00:00:00 2001 From: Noah Date: Wed, 27 Aug 2025 16:54:30 +1000 Subject: [PATCH 19/27] modified: module/netbox/object_classes.py --- module/netbox/object_classes.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 0b954a61..b88efb24 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2150,18 +2150,6 @@ def resolve_relations(self): def update(self, data=None, read_from_netbox=False, source=None): object_type = data.get("assigned_object_type") assigned_object = data.get("assigned_object_id") - - # Skip IP assignments when the IP is assigned to FHRP groups when config option - # skip_fhrp_group_ips is set to True, or if the IP is manually assigned to an FHRP group (no source) - if source is not None: - if source.source_type == "vmware": - config_relation = source.get_object_relation(assigned_object, "skip_fhrp_group_ips") - if config_relation == True and object_type == "ipam.fhrpgroup": - log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") - return - elif object_type == "ipam.fhrpgroup": - log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. It was manually created. Skipping.") - return # used to track changes in object primary IP assignments previous_ip_device_vm = None @@ -2173,6 +2161,19 @@ def update(self, data=None, read_from_netbox=False, source=None): # get current device to make sure to unset primary ip before moving IP address previous_ip_device_vm = self.get_device_vm() + + # Skip IP assignments when the IP is already assigned to FHRP groups when config option + # skip_fhrp_group_ips is set to True, or if the IP is manually assigned to an FHRP group (no source) + if source is not None: + if source.source_type == "vmware": + config_relation = source.get_object_relation(assigned_object, "skip_fhrp_group_ips") + if config_relation == True and grab(previous_ip_device_vm, "data.assigned_object_type") == "ipam.fhrpgroup": + log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") + return + elif grab(previous_ip_device_vm, "data.assigned_object_type") == "ipam.fhrpgroup": + log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. It was manually created. Skipping.") + return + if grab(previous_ip_device_vm, "data.primary_ip4") is self: is_primary_ipv4_of_previous_device = True if grab(previous_ip_device_vm, "data.primary_ip6") is self: @@ -2213,7 +2214,7 @@ def get_interface(self): o_id = self.data.get("assigned_object_id") o_type = self.data.get("assigned_object_type") - if isinstance(o_id, (NBInterface, NBVMInterface)): + if isinstance(o_id, (NBInterface, NBVMInterface, NBFHRPGroupItem)): return o_id if o_type is None or not isinstance(o_id, int): @@ -2235,6 +2236,8 @@ def get_device_vm(self): return o_interface.data.get("device") elif isinstance(o_interface, NBVMInterface): return o_interface.data.get("virtual_machine") + elif isinstance(o_interface, NBFHRPGroupItem): + return o_interface.data.get("fhrp_group") def remove_interface_association(self): o_id = self.data.get("assigned_object_id") From 3a22fc2e743a2bb2397e021403ddc013152e1b53 Mon Sep 17 00:00:00 2001 From: Noah Date: Thu, 28 Aug 2025 10:30:47 +1000 Subject: [PATCH 20/27] Undid part of a previous commit, to undo a mistake --- module/netbox/object_classes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index b88efb24..45530a20 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2167,10 +2167,10 @@ def update(self, data=None, read_from_netbox=False, source=None): if source is not None: if source.source_type == "vmware": config_relation = source.get_object_relation(assigned_object, "skip_fhrp_group_ips") - if config_relation == True and grab(previous_ip_device_vm, "data.assigned_object_type") == "ipam.fhrpgroup": + if config_relation == True and object_type == "ipam.fhrpgroup": log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") return - elif grab(previous_ip_device_vm, "data.assigned_object_type") == "ipam.fhrpgroup": + elif object_type == "ipam.fhrpgroup": log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. It was manually created. Skipping.") return From 3d9a189fa0fb8383116761c863a49678444dcb9a Mon Sep 17 00:00:00 2001 From: Noah Date: Thu, 28 Aug 2025 15:56:12 +1000 Subject: [PATCH 21/27] Added a dev document for explaining how the code works for the benefit of other devs --- docs/dev-sync-process.md | 76 +++++++++++++++++++++++++++++ module/sources/vmware/connection.py | 4 +- 2 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 docs/dev-sync-process.md diff --git a/docs/dev-sync-process.md b/docs/dev-sync-process.md new file mode 100644 index 00000000..9ba4d187 --- /dev/null +++ b/docs/dev-sync-process.md @@ -0,0 +1,76 @@ +This document is intended to be an overview of how the code is working. + +1. source list is validated. +2. sources are instantiated. + the sources are returned as a source handler list. +3. netbox data is queried and cached + All data types in NBInventory class are: + "FHRP group": [], + "IP address": [], + "IP prefix": [], + "MAC address": [], + "VLAN": [], + "VLANGroup": [], + "VRF": [], + "Virtual Disk": [], + "cluster": [], + "cluster group": [], + "cluster type": [], + "custom field": [], + "device": [], + "device role": [], + "device type": [], + "interface": [], + "inventory item": [], + "manufacturer": [], + "platform": [], + "power port": [], + "site": [], + "site group": [], + "tag": [], + "tenant": [], + "virtual machine": [], + "virtual machine interface": [] + these are all of the possible, valid inventory entries. obtained from adding 'log.debug(inventory)' to module/sources/__init__.py, line 84. +4. source handler 'apply()' method is called, which retrieves vmware data. In this method, 'view handers' are called, which individually add each data type (e.g. data center, cluster) to the 'object_mapping' dict. + object_mapping entries are: + "datacenter": { + "view_type": vim.Datacenter, + "view_handler": self.add_datacenter + }, + "cluster": { + "view_type": vim.ClusterComputeResource, + "view_handler": self.add_cluster + }, + "single host cluster": { + "view_type": vim.ComputeResource, + "view_handler": self.add_cluster + }, + "network": { + "view_type": vim.dvs.DistributedVirtualPortgroup, + "view_handler": self.add_port_group + }, + "host": { + "view_type": vim.HostSystem, + "view_handler": self.add_host + }, + "virtual machine": { + "view_type": vim.VirtualMachine, + "view_handler": self.add_virtual_machine + }, + "offline virtual machine": { + "view_type": vim.VirtualMachine, + "view_handler": self.add_virtual_machine + } +5. the queried data is added to the cache under the view handler methods. + the view handler methods are responsible for adding all the individual data to the cache and inventory +6. in the vm view handler, all the extra data from the vm's is collected and parsed, and then passed into the add_device_vm_to_inventory method. currently i believe this is where the ip address data is coming from. the params for the above method are: + NBVM, object_data=vm_data, vnic_data=nic_data, nic_ips=nic_ips, p_ipv4=vm_primary_ip4, p_ipv6=vm_primary_ip6, vmware_object=obj, disk_data=disk_data +7. in this file /module/sources/vmware/connection.py, line 956 is the method add_device_vm_to_inventory. this is where vm's are added to the inventory, and the corresponding objects (vm attributes, like site, device, primary ip address) are matched or created. +8. after looking through the add_update_interface method (/module/sources/common/source_base.py, line 234) within the previous method, vmware does not appear to be providing any fhrp group related data. itRequest logs is possible this is because i don't have any fhrp group data to sync. + interface data provided (in the form ([ip address], {assigned object})): + ([], {'name': 'vmnic0', 'device': None, 'mac_address': '0C:4D:E9:99:EE:51', 'enabled': True, 'description': '1Gb/s pNIC (vSwitch0)', 'type': '1000base-t', 'mtu': 1500, 'speed': 1000000, 'duplex': 'full', 'mode': 'access'}) + (['192.168.11.143/255.255.255.0'], {'name': 'vmk0', 'device': None, 'mac_address': '0C:4D:E9:99:EE:51', 'enabled': True, 'mtu': 1500, 'type': 'virtual', 'mode': 'access', 'description': 'Management Network (vSwitch0, vlan ID: 0)'}) + (['192.168.11.147/24'], {'name': 'vNIC 1 (VM Network)', 'virtual_machine': None, 'mac_address': '00:0C:29:F3:3C:69', 'description': 'Network adapter 1 (VirtualVmxnet3) (vlan ID: 0)', 'enabled': True, 'mtu': 1500, 'mode': 'access'}) + this method is where ip address objects are created and added to the inventory if new and updated if not. + the method also returns the ip addresses as a list (as well as the interface object) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 542f08a2..553a724d 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -909,8 +909,8 @@ def get_object_relation(self, name, relation, fallback=None): Returns ------- - data: str, list, None - string of matching relation or list of matching tags + data: str, list, bool, None + string of matching relation or list of matching tags, or boolean if relation is boolean """ resolved_list = list() From 3f5af8c8c9a6b00564347d24f9091fb22462007e Mon Sep 17 00:00:00 2001 From: Noah Date: Thu, 28 Aug 2025 16:59:54 +1000 Subject: [PATCH 22/27] Added/changed where the fhrp group ip address updating prevention occured to prevent a duplicate ip address from being created, causing errors. --- module/netbox/object_classes.py | 12 ------------ module/sources/common/source_base.py | 6 ++++++ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 45530a20..ed0add6f 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2162,18 +2162,6 @@ def update(self, data=None, read_from_netbox=False, source=None): # get current device to make sure to unset primary ip before moving IP address previous_ip_device_vm = self.get_device_vm() - # Skip IP assignments when the IP is already assigned to FHRP groups when config option - # skip_fhrp_group_ips is set to True, or if the IP is manually assigned to an FHRP group (no source) - if source is not None: - if source.source_type == "vmware": - config_relation = source.get_object_relation(assigned_object, "skip_fhrp_group_ips") - if config_relation == True and object_type == "ipam.fhrpgroup": - log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. Skipping.") - return - elif object_type == "ipam.fhrpgroup": - log.debug(f"IP address with id '{assigned_object}' assigned to FHRP group. It was manually created. Skipping.") - return - if grab(previous_ip_device_vm, "data.primary_ip4") is self: is_primary_ipv4_of_previous_device = True if grab(previous_ip_device_vm, "data.primary_ip6") is self: diff --git a/module/sources/common/source_base.py b/module/sources/common/source_base.py index 0e95e5cc..7611668c 100644 --- a/module/sources/common/source_base.py +++ b/module/sources/common/source_base.py @@ -443,6 +443,12 @@ def add_update_interface(self, interface_object, device_object, interface_data, this_ip_object = None skip_this_ip = False for ip in self.inventory.get_all_items(NBIPAddress): + # stops fhrp group assigned ip addresses from being overridden and assigned to another object type + # if the config skip_fhrp_group_ips is set to True + if grab(ip, "data.assigned_object_type", fallback="") == "ipam.fhrpgroup" and self.settings.skip_fhrp_group_ips: + log.info(f"Ip address {grab(ip, "data.address")} is assigned to an FHRP Group and skip_fhrp_group_ips is set to {self.settings.skip_fhrp_group_ips}, skipping.") + skip_this_ip = True + continue # check if address matches (without prefix length) ip_address_string = grab(ip, "data.address", fallback="") From a7acdac1b7aaedb90cf82cb9452b31bf54777534 Mon Sep 17 00:00:00 2001 From: Noah Date: Fri, 29 Aug 2025 09:05:14 +1000 Subject: [PATCH 23/27] removed the dev doc since it's not related to this bug --- docs/dev-sync-process.md | 76 ---------------------------------------- 1 file changed, 76 deletions(-) delete mode 100644 docs/dev-sync-process.md diff --git a/docs/dev-sync-process.md b/docs/dev-sync-process.md deleted file mode 100644 index 9ba4d187..00000000 --- a/docs/dev-sync-process.md +++ /dev/null @@ -1,76 +0,0 @@ -This document is intended to be an overview of how the code is working. - -1. source list is validated. -2. sources are instantiated. - the sources are returned as a source handler list. -3. netbox data is queried and cached - All data types in NBInventory class are: - "FHRP group": [], - "IP address": [], - "IP prefix": [], - "MAC address": [], - "VLAN": [], - "VLANGroup": [], - "VRF": [], - "Virtual Disk": [], - "cluster": [], - "cluster group": [], - "cluster type": [], - "custom field": [], - "device": [], - "device role": [], - "device type": [], - "interface": [], - "inventory item": [], - "manufacturer": [], - "platform": [], - "power port": [], - "site": [], - "site group": [], - "tag": [], - "tenant": [], - "virtual machine": [], - "virtual machine interface": [] - these are all of the possible, valid inventory entries. obtained from adding 'log.debug(inventory)' to module/sources/__init__.py, line 84. -4. source handler 'apply()' method is called, which retrieves vmware data. In this method, 'view handers' are called, which individually add each data type (e.g. data center, cluster) to the 'object_mapping' dict. - object_mapping entries are: - "datacenter": { - "view_type": vim.Datacenter, - "view_handler": self.add_datacenter - }, - "cluster": { - "view_type": vim.ClusterComputeResource, - "view_handler": self.add_cluster - }, - "single host cluster": { - "view_type": vim.ComputeResource, - "view_handler": self.add_cluster - }, - "network": { - "view_type": vim.dvs.DistributedVirtualPortgroup, - "view_handler": self.add_port_group - }, - "host": { - "view_type": vim.HostSystem, - "view_handler": self.add_host - }, - "virtual machine": { - "view_type": vim.VirtualMachine, - "view_handler": self.add_virtual_machine - }, - "offline virtual machine": { - "view_type": vim.VirtualMachine, - "view_handler": self.add_virtual_machine - } -5. the queried data is added to the cache under the view handler methods. - the view handler methods are responsible for adding all the individual data to the cache and inventory -6. in the vm view handler, all the extra data from the vm's is collected and parsed, and then passed into the add_device_vm_to_inventory method. currently i believe this is where the ip address data is coming from. the params for the above method are: - NBVM, object_data=vm_data, vnic_data=nic_data, nic_ips=nic_ips, p_ipv4=vm_primary_ip4, p_ipv6=vm_primary_ip6, vmware_object=obj, disk_data=disk_data -7. in this file /module/sources/vmware/connection.py, line 956 is the method add_device_vm_to_inventory. this is where vm's are added to the inventory, and the corresponding objects (vm attributes, like site, device, primary ip address) are matched or created. -8. after looking through the add_update_interface method (/module/sources/common/source_base.py, line 234) within the previous method, vmware does not appear to be providing any fhrp group related data. itRequest logs is possible this is because i don't have any fhrp group data to sync. - interface data provided (in the form ([ip address], {assigned object})): - ([], {'name': 'vmnic0', 'device': None, 'mac_address': '0C:4D:E9:99:EE:51', 'enabled': True, 'description': '1Gb/s pNIC (vSwitch0)', 'type': '1000base-t', 'mtu': 1500, 'speed': 1000000, 'duplex': 'full', 'mode': 'access'}) - (['192.168.11.143/255.255.255.0'], {'name': 'vmk0', 'device': None, 'mac_address': '0C:4D:E9:99:EE:51', 'enabled': True, 'mtu': 1500, 'type': 'virtual', 'mode': 'access', 'description': 'Management Network (vSwitch0, vlan ID: 0)'}) - (['192.168.11.147/24'], {'name': 'vNIC 1 (VM Network)', 'virtual_machine': None, 'mac_address': '00:0C:29:F3:3C:69', 'description': 'Network adapter 1 (VirtualVmxnet3) (vlan ID: 0)', 'enabled': True, 'mtu': 1500, 'mode': 'access'}) - this method is where ip address objects are created and added to the inventory if new and updated if not. - the method also returns the ip addresses as a list (as well as the interface object) From 11ed756709bfba7e2d1ac78ecb5abdbb4e756426 Mon Sep 17 00:00:00 2001 From: Noah Date: Fri, 29 Aug 2025 09:22:16 +1000 Subject: [PATCH 24/27] Removed the last of the unnecessary things (hopefully) --- module/netbox/object_classes.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index ed0add6f..29f72a60 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2148,6 +2148,7 @@ def resolve_relations(self): super().resolve_relations() def update(self, data=None, read_from_netbox=False, source=None): + object_type = data.get("assigned_object_type") assigned_object = data.get("assigned_object_id") @@ -2202,7 +2203,7 @@ def get_interface(self): o_id = self.data.get("assigned_object_id") o_type = self.data.get("assigned_object_type") - if isinstance(o_id, (NBInterface, NBVMInterface, NBFHRPGroupItem)): + if isinstance(o_id, (NBInterface, NBVMInterface)): return o_id if o_type is None or not isinstance(o_id, int): @@ -2224,8 +2225,6 @@ def get_device_vm(self): return o_interface.data.get("device") elif isinstance(o_interface, NBVMInterface): return o_interface.data.get("virtual_machine") - elif isinstance(o_interface, NBFHRPGroupItem): - return o_interface.data.get("fhrp_group") def remove_interface_association(self): o_id = self.data.get("assigned_object_id") From bd3d882f43cebf0ee54bded7e193e2d1ad2c71fb Mon Sep 17 00:00:00 2001 From: Noah Date: Fri, 29 Aug 2025 09:25:23 +1000 Subject: [PATCH 25/27] one last tiny thing --- module/netbox/object_classes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/module/netbox/object_classes.py b/module/netbox/object_classes.py index 29f72a60..9d250935 100644 --- a/module/netbox/object_classes.py +++ b/module/netbox/object_classes.py @@ -2162,7 +2162,6 @@ def update(self, data=None, read_from_netbox=False, source=None): # get current device to make sure to unset primary ip before moving IP address previous_ip_device_vm = self.get_device_vm() - if grab(previous_ip_device_vm, "data.primary_ip4") is self: is_primary_ipv4_of_previous_device = True if grab(previous_ip_device_vm, "data.primary_ip6") is self: From 409642d5a28a53392faf1fce30409cd260491844 Mon Sep 17 00:00:00 2001 From: Noah Date: Fri, 29 Aug 2025 13:33:39 +1000 Subject: [PATCH 26/27] cleanup unnecessary code --- module/sources/vmware/connection.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/module/sources/vmware/connection.py b/module/sources/vmware/connection.py index 553a724d..742f54ca 100644 --- a/module/sources/vmware/connection.py +++ b/module/sources/vmware/connection.py @@ -909,17 +909,11 @@ def get_object_relation(self, name, relation, fallback=None): Returns ------- - data: str, list, bool, None - string of matching relation or list of matching tags, or boolean if relation is boolean + data: str, list, None + string of matching relation or list of matching tags """ resolved_list = list() - relation_data = grab(self.settings, relation, fallback=fallback) - - if isinstance(relation_data, bool): - log.debug(f"Object relation '{relation}' is boolean, set '{relation_data}'.") - return relation_data - for single_relation in grab(self.settings, relation, fallback=list()): object_regex = single_relation.get("object_regex") match_found = False From 1e865b7e7a446dfaf5fc7d6fc85d806bd41800c8 Mon Sep 17 00:00:00 2001 From: Noah Date: Fri, 29 Aug 2025 13:45:58 +1000 Subject: [PATCH 27/27] removed the readme note not in the dev branch --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index b944ac26..0430a672 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,6 @@ # NetBox-Sync -> [!CAUTION] -> **Maintainer wanted - sunsetting this repository by 31.10.2025 [#474](https://github.com/bb-Ricardo/netbox-sync/issues/474)** - This is a tool to sync data from different sources to a NetBox instance. Available source types: