From 51bd3b7b22614c8248ff316347626084724518c2 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Mon, 23 May 2022 23:40:34 -0400 Subject: [PATCH 01/41] continue adding the setup param to utility functions --- netfoundry/organization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netfoundry/organization.py b/netfoundry/organization.py index 49b8b4c..125eaa5 100644 --- a/netfoundry/organization.py +++ b/netfoundry/organization.py @@ -153,7 +153,7 @@ def __init__(self, # if the token was found but not the expiry then try to parse to extract the expiry so we can enforce minimum lifespan seconds if self.token and not self.expiry: try: - self.expiry = jwt_expiry(self) + self.expiry = jwt_expiry(setup=self) except Exception as e: self.expiry = round(epoch + DEFAULT_TOKEN_EXPIRY) self.expiry_seconds = DEFAULT_TOKEN_EXPIRY @@ -243,7 +243,7 @@ def __init__(self, # the try-except block is to soft-fail all attempts to parse the JWT, # which is intended for the API, not this application if self.token and not self.environment: - self.environment = jwt_environment(self) + self.environment = jwt_environment(setup=self) self.logger.debug(f"parsed token as JWT and found environment {self.environment}") if self.environment not in ENVIRONMENTS: self.logger.warning(f"unexpected environment '{self.environment}'") From 717f2f5c77f7c973c50714997d2882416d6f52cf Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Mon, 23 May 2022 23:40:58 -0400 Subject: [PATCH 02/41] start deprecating download-urls --- netfoundry/utility.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/netfoundry/utility.py b/netfoundry/utility.py index a02fa04..d1fc672 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -241,8 +241,8 @@ def get_resource_type_by_url(url: str): url_parts = urlparse(url) url_path = url_parts.path resource_type = re.sub(r'/(core|rest|identity|auth|product-metadata)/v\d+/([^/]+)/?.*', r'\2', url_path) - if resource_type == "download-urls.json": - resource_type = "download-urls" + # if resource_type == "download-urls.json": + # resource_type = "download-urls" if RESOURCES.get(resource_type): return RESOURCES.get(resource_type) else: @@ -729,13 +729,13 @@ def __post_init__(self): embeddable=False, find_url="iframe.php?url=https%3A%2F%2Fgithub.com%2Frest%2Fv1%2Fnetwork-groups", ), - 'download-urls': ResourceType( - name='download-urls', - domain='network-group', - mutable=False, - embeddable=False, - find_url="iframe.php?url=https%3A%2F%2Fgithub.com%2Fproduct-metadata%2Fv2", - ), + # 'download-urls': ResourceType( + # name='download-urls', + # domain='network-group', + # mutable=False, + # embeddable=False, + # find_url="iframe.php?url=https%3A%2F%2Fgithub.com%2Fproduct-metadata%2Fv2", + # ), 'networks': ResourceType( name='networks', domain='network', From ba553123a8edf9f3faed2ad35deac34df58008f1 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Mon, 23 May 2022 23:41:16 -0400 Subject: [PATCH 03/41] restore the legacy Utility interface --- netfoundry/utility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netfoundry/utility.py b/netfoundry/utility.py index d1fc672..5a6d160 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -7,7 +7,7 @@ import unicodedata # case insensitive compare in Utility from dataclasses import dataclass, field from json import JSONDecodeError -from re import sub +# from re import sub from stat import filemode from urllib.parse import urlparse from uuid import UUID # validate UUIDv4 strings From ddbc9fd4b69dbc1a5c640787f8c1a94941f8b2e3 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 24 May 2022 01:17:02 -0400 Subject: [PATCH 04/41] decommission "data centers" in favor of "regions" --- netfoundry/ctl.py | 21 +++++------ netfoundry/network.py | 73 ++++++++++--------------------------- netfoundry/network_group.py | 58 ++++++++++------------------- netfoundry/utility.py | 21 ----------- 4 files changed, 47 insertions(+), 126 deletions(-) diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index 1f7a8d8..3a72af2 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -523,17 +523,17 @@ def get(cli, echo: bool = True, spinner: object = None): else: cli.log.error("need --network=ACMENet") sysexit(1) - if cli.args.resource_type == "data-center": + if cli.args.resource_type == "region": if 'id' in query_keys: - cli.log.warn("data centers fetched by ID may not support this network's product version, try provider or locationCode params for safety") + cli.log.warn("regions fetched by ID may not support this network's product version, try provider or locationCode params for safety") if len(query_keys) > 1: query_keys.remove('id') cli.log.warn(f"using 'id' only, ignoring params: '{', '.join(query_keys)}'") - match = network.get_data_center_by_id(id=cli.args.query['id']) + match = network.get_region_by_id(id=cli.args.query['id']) else: - matches = network.find_edge_router_data_centers(**cli.args.query) + matches = networks.find_regions(**cli.args.query) if len(matches) == 1: - match = network.get_data_center_by_id(id=matches[0]['id']) + match = network.get_region_by_id(id=matches[0]['id']) else: if 'id' in query_keys: if len(query_keys) > 1: @@ -618,7 +618,7 @@ def list(cli, echo: bool = True, spinner: object = None): cli.log.debug("got spinner as function param") if RESOURCE_ABBREV.get(cli.args.resource_type): cli.args.resource_type = RESOURCE_ABBREV[cli.args.resource_type].name - if cli.args.accept and not MUTABLE_NET_RESOURCES.get(cli.args.resource_type): # mutable excludes data-centers + if cli.args.accept and not MUTABLE_NET_RESOURCES.get(cli.args.resource_type): cli.log.warn("the --as=ACCEPT param is not applicable to resources outside the network domain") if cli.args.query and cli.args.query.get('id'): cli.log.warn("try 'get' command to get by id") @@ -676,10 +676,7 @@ def list(cli, echo: bool = True, spinner: object = None): else: cli.log.error("first configure a network: '--network=ACMENet'") sysexit(1) - if cli.args.resource_type == "data-centers": - matches = network.find_edge_router_data_centers(**cli.args.query) - else: - matches = network.find_resources(type=cli.args.resource_type, accept=cli.args.accept, params=cli.args.query) + matches = network.find_resources(type=cli.args.resource_type, accept=cli.args.accept, params=cli.args.query) if len(matches) == 0: spinner.fail(f"Found no {cli.args.resource_type} by '{', '.join(query_keys)}'") @@ -978,8 +975,8 @@ def demo(cli): # a list of locations to place a hosted router fabric_placements = [] for region in cli.config.demo.regions: - dc_matches = network.find_edge_router_data_centers(provider=cli.config.demo.provider, location_code=region) - if not len(dc_matches) == 1: + region_matches = networks.find_regions(provider=cli.config.demo.provider, location_code=region) + if not len(region_matches) == 1: raise RuntimeError(f"invalid region '{region}'") else: existing_count = len([er for er in hosted_edge_routers if er['provider'] == cli.config.demo.provider and er['region'] == region]) diff --git a/netfoundry/network.py b/netfoundry/network.py index 37c1fdc..5519b5a 100644 --- a/netfoundry/network.py +++ b/netfoundry/network.py @@ -265,59 +265,25 @@ def validate_entity_roles(self, entities: list, type: str): return(valid_entities) - def get_data_center_by_id(self, id: str): - """Get data centers by UUIDv4. + def get_region_by_id(self, id: str): + """Get data center region by UUIDv4. :param id: required UUIDv4 of data center """ - url = self.audience+'core/v2/data-centers/'+id - headers = {"authorization": "Bearer " + self.token} - try: - data_center, status_symbol = get_generic_resource_by_url(url=url, headers=headers, proxies=self.proxies, verify=self.verify) - except Exception as e: - raise RuntimeError(f"failed to get data_center from url: '{url}', caught {e}") - else: - return(data_center) + url = self.audience+'core/v2/regions/'+id + data_center, status_symbol = get_generic_resource_by_url(setup=self, url=url) + return(data_center) + get_data_center_by_id = get_region_by_id @docstring_parameters(providers=str(DC_PROVIDERS)) - def find_edge_router_data_centers(self, provider: str = None, location_code: str = None, **kwargs): - """Find data centers for hosting edge routers. + def find_regions(self, **kwargs): + """Find regions for hosted router placement. - :param provider: optionally filter by data center provider, choices: {providers} - :param location_code: provider-specific string identifying the data center location e.g. us-west-1 + :param provider: optionally filter by data-center region provider, choices: {providers} """ - params = dict() - for param in kwargs.keys(): - if param == 'region': - location_code = kwargs[param] - else: - params[param] = kwargs[param] - if not params.get('productVersion'): - params["productVersion"] = self.product_version - if not params.get('hostType'): - params["hostType"] = "ER" - - if location_code: - params["locationCode"] = location_code - elif params.get('locationCode'): - location_code = params['locationCode'] # query param not yet implemented in API so store it in function to filter the list in response - elif params.get('region'): - location_code = params['region'] # alternatively, get the location_code from a 'region' query param - if provider is not None: - if provider in DC_PROVIDERS: - params['provider'] = provider - else: - raise RuntimeError(f"unknown cloud provider '{provider}'. Need one of {str(DC_PROVIDERS)}") - - url = self.audience+'core/v2/data-centers' - data_centers = list() - for i in find_generic_resources(setup=self, url=url, embedded=NET_RESOURCES['data-centers']._embedded, **params): - data_centers.extend(i) - if location_code: - return [dc for dc in data_centers if dc['locationCode'] == location_code] - else: - return data_centers - get_edge_router_data_centers = find_edge_router_data_centers + return self.Networks.find_regions(**kwargs) + get_edge_router_data_centers = find_regions + find_edge_router_data_centers = find_regions def share_endpoint(self, recipient, endpoint_id): """ @@ -1687,12 +1653,11 @@ def find_regions(self, **kwargs): for param in kwargs.keys(): params[param] = kwargs[param] + if params.get('provider') and not params['provider'] in DC_PROVIDERS: + raise RuntimeError(f"unknown cloud provider '{params['provider']}'. Need one of {str(DC_PROVIDERS)}") + url = self.audience+NET_RESOURCES['regions'].find_url - try: - regions = list() - for i in find_generic_resources(setup=self, url=url, embedded=NET_RESOURCES['data-centers']._embedded, **params): - regions.extend(i) - except Exception as e: - raise RuntimeError(f"failed to get data-centers from url: '{url}', caught {e}") - else: - return(regions) + regions = list() + for i in find_generic_resources(setup=self, url=url, embedded=NET_RESOURCES['regions']._embedded, **params): + regions.extend(i) + return(regions) diff --git a/netfoundry/network_group.py b/netfoundry/network_group.py index b5b02b9..98d60cb 100644 --- a/netfoundry/network_group.py +++ b/netfoundry/network_group.py @@ -13,7 +13,6 @@ class NetworkGroup: def __init__(self, Organization: object, network_group_id: str = None, network_group_name: str = None, group: str = None): """Initialize the network group class with a group name or ID.""" self.logger = Organization.logger - self.Networks = Networks(Organization) self.network_groups = Organization.find_network_groups_by_organization() if (not network_group_id and not network_group_name) and group: self.logger.debug(f"got 'group' = '{group}' which could be a short name or id") @@ -70,13 +69,14 @@ def __init__(self, Organization: object, network_group_id: str = None, network_g for net in Organization.get_networks_by_group(network_group_id=self.network_group_id): self.network_ids_by_normal_name[normalize_caseless(net['name'])] = net['id'] - def nc_data_centers_by_location(self): - """Get a controller data center by locationCode.""" - my_nc_data_centers_by_location = dict() - for dc in self.get_controller_data_centers(): - my_nc_data_centers_by_location[dc['locationCode']] = dc['id'] + def map_region_id_by_location_code(self): + """Map all region ids by their location code e.g. us-west-1: id.""" + region_map = dict() + for region in Networks.find_regions(provider='OCP') + Networks.find_regions(provider='AWS'): + region_map[region['locationCode']] = region['id'] # e.g. { us-east-1: 02f0eb51-fb7a-4d2e-8463-32bd9f6fa4d7 } - return(my_nc_data_centers_by_location) + return(region_map) + nc_data_centers_by_location = map_region_id_by_location_code # resolve network UUIDs by name def network_id_by_normal_name(self, name): @@ -101,26 +101,10 @@ def network_exists(self, name: str, deleted: bool = False): else: return(False) - def nc_data_centers(self, **kwargs): - """Find network controller data centers.""" - # data centers returns a list of dicts (data center objects) - params = dict() - for param in kwargs.keys(): - params[param] = kwargs[param] - params["productVersion"] = self.find_latest_network_version(is_active=True) - params["hostType"] = "NC" - params["provider"] = "AWS" - - url = self.audience+'core/v2/data-centers' - headers = {"authorization": "Bearer " + self.token} - try: - data_centers = list() - for i in find_generic_resources(setup=self, url=url, embedded=NET_RESOURCES['data-centers']._embedded, **params): - data_centers.extend(i) - except Exception as e: - raise RuntimeError(f"failed to get data-centers from url: '{url}', caught {e}") - else: - return(data_centers) + def find_regions(self, **kwargs): + """Find network controller data center regions.""" + return(Networks.find_regions(**kwargs)) + nc_data_centers = find_regions # provide a compatible alias get_controller_data_centers = nc_data_centers @@ -133,19 +117,15 @@ def get_product_metadata(self, is_active: bool = True): :param product_version: semver string of a single version to get, default is all versions """ url = self.audience+'product-metadata/v2/download-urls.json' - try: - all_product_metadata, status_symbol = get_generic_resource_by_url(setup=self, url=url) - except Exception as e: - raise RuntimeError(f"failed to get product-metadata from url: '{url}', caught {e}") + all_product_metadata, status_symbol = get_generic_resource_by_url(setup=self, url=url) + if is_active: + filtered_product_metadata = dict() + for product in all_product_metadata.keys(): + if all_product_metadata[product]['active']: + filtered_product_metadata[product] = all_product_metadata[product] + return (filtered_product_metadata) else: - if is_active: - filtered_product_metadata = dict() - for product in all_product_metadata.keys(): - if all_product_metadata[product]['active']: - filtered_product_metadata[product] = all_product_metadata[product] - return (filtered_product_metadata) - else: - return (all_product_metadata) + return (all_product_metadata) def find_network_versions(self, is_active: bool = True): """Find active network versions.""" diff --git a/netfoundry/utility.py b/netfoundry/utility.py index 5a6d160..188448d 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -687,13 +687,6 @@ def __post_init__(self): status_symbols=PROCESS_STATUS_SYMBOLS, abbreviation='ex', ), - # 'processes': ResourceType( # not a fully-fledged resource type because there is no "find processes" operation at this time - # name='processes', - # domain='network', - # mutable=False, - # embeddable=False, - # status_symbols=PROCESS_STATUS_SYMBOLS, - # ), 'regions': ResourceType( name='regions', domain='network', @@ -708,13 +701,6 @@ def __post_init__(self): embeddable=False, _embedded='network-versions', ), - 'data-centers': ResourceType( - name='data-centers', - domain='network', - _embedded='dataCenters', - mutable=False, - embeddable=False, - ), 'organizations': ResourceType( name='organizations', domain='identity', @@ -729,13 +715,6 @@ def __post_init__(self): embeddable=False, find_url="iframe.php?url=https%3A%2F%2Fgithub.com%2Frest%2Fv1%2Fnetwork-groups", ), - # 'download-urls': ResourceType( - # name='download-urls', - # domain='network-group', - # mutable=False, - # embeddable=False, - # find_url="iframe.php?url=https%3A%2F%2Fgithub.com%2Fproduct-metadata%2Fv2", - # ), 'networks': ResourceType( name='networks', domain='network', From ee9310b041882c6176bd54e81f9edde1fd1f5754 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 24 May 2022 01:17:59 -0400 Subject: [PATCH 05/41] no cache when getting things that need to be evergreen, like statuses --- netfoundry/network.py | 62 ++++++++++++++++++++----------------------- netfoundry/utility.py | 35 ++++++++++++++---------- 2 files changed, 50 insertions(+), 47 deletions(-) diff --git a/netfoundry/network.py b/netfoundry/network.py index 5519b5a..8f94a7e 100644 --- a/netfoundry/network.py +++ b/netfoundry/network.py @@ -31,6 +31,8 @@ def __init__(self, NetworkGroup: object, network_id: str = None, network_name: s self.audience = NetworkGroup.audience self.network_group_id = NetworkGroup.id + self.Networks = Networks(NetworkGroup) + if (not network_id and not network_name) and network: if is_uuidv4(network): network_id = network @@ -317,7 +319,7 @@ def share_endpoint(self, recipient, endpoint_id): if not response_code == STATUS_CODES.codes['ACCEPTED']: raise RuntimeError(f"got unexpected HTTP code {STATUS_CODES._codes[response_code][0].upper()} ({response_code}) and response {response.text}") - def get_resource_by_id(self, type: str, id: str, accept: str = None): + def get_resource_by_id(self, type: str, id: str, accept: str = None, use_cache: bool=True): """Return an object describing a resource entity. :param type: required string of the singular of an entity type e.g. network, endpoint, service, edge-router, edge-router-policy, posture-check @@ -341,13 +343,13 @@ def get_resource_by_id(self, type: str, id: str, accept: str = None): raise RuntimeError(f"unknown resource type '{plural(type)}'. Choices: {', '.join(NET_RESOURCES.keys())}") url = self.audience+'core/v2/'+plural(type)+'/'+id - resource, status_symbol = get_generic_resource_by_url(setup=self, url=url, headers=headers) + resource, status_symbol = get_generic_resource_by_url(setup=self, url=url, headers=headers, use_cache=use_cache) if not resource['networkId'] == self.id: raise NetworkBoundaryViolation("resource ID is from another network") return(resource) get_resource = get_resource_by_id - def find_resources(self, type: str, accept: str = None, deleted: bool = False, params: dict = dict(), **kwargs): + def find_resources(self, type: str, accept: str = None, deleted: bool = False, params: dict = dict(), use_cache: bool = True, **kwargs): """Find resources by type. :param str type: plural of an entity type in the network domain e.g. networks, endpoints, services, posture-checks, etc... @@ -360,8 +362,6 @@ def find_resources(self, type: str, accept: str = None, deleted: bool = False, p # pluralize if singular if not type[-1] == "s": type = plural(type) - if type == "data-centers": - self.logger.warning("don't call network.get_resources() for data centers, always use network.get_edge_router_data_centers() to filter for locations that support this network's version") for param in kwargs.keys(): params[param] = kwargs[param] @@ -376,7 +376,7 @@ def find_resources(self, type: str, accept: str = None, deleted: bool = False, p url = self.audience+'core/v2/'+plural(type) try: resources = list() - for i in find_generic_resources(setup=self, url=url, embedded=NET_RESOURCES[plural(type)]._embedded, accept=accept, **params): + for i in find_generic_resources(setup=self, url=url, embedded=NET_RESOURCES[plural(type)]._embedded, accept=accept, use_cache=use_cache, **params): resources.extend(i) except Exception as e: raise RuntimeError(f"failed to get {plural(type)} from url: '{url}', caught {e}") @@ -424,7 +424,7 @@ def patch_resource(self, patch: dict, type: str = None, id: str = None, wait: in if not MUTABLE_NET_RESOURCES.get(type): # prune properties that can't be patched raise RuntimeError(f"got unexpected type '{type}' for patch request to {self_link}") - before_resource, status_symbol = get_generic_resource_by_url(setup=self, url=self_link) + before_resource, status_symbol = get_generic_resource_by_url(setup=self, url=self_link, use_cache=False) self.logger.debug(f"found existing resource before patching with properties: '{before_resource}'") # compare the patch to the discovered, current state, adding new or updated keys to pruned_patch pruned_patch = dict() @@ -568,7 +568,7 @@ def create_resource(self, type: str, post: dict, wait: int = 30, sleep: int = 2, if body.get('name'): body['name'] = body['name'].strip('"') url = self.audience+'core/v2/'+type - resource = create_generic_resource(url=url, body=body, headers=headers, proxies=self.proxies, verify=self.verify, wait=wait, sleep=sleep) + resource = create_generic_resource(setup=self, url=url, body=body, wait=wait, sleep=sleep) return(resource) def create_endpoint(self, name: str, attributes: list = [], session_identity: str = None, wait: int = 30, sleep: int = 2, progress: bool = False): @@ -596,7 +596,7 @@ def create_endpoint(self, name: str, attributes: list = [], session_identity: st body['sessionIdentityId'] = session_identity url = self.audience+'core/v2/endpoints' - resource = create_generic_resource(url=url, body=body, headers=headers, proxies=self.proxies, verify=self.verify, wait=wait, sleep=sleep) + resource = create_generic_resource(setup=self, url=url, body=body, wait=wait, sleep=sleep) self.logger.debug(f"created endpoint {resource['name']}") return(resource) @@ -651,10 +651,10 @@ def create_edge_router(self, name: str, attributes: list = list(), link_listener raise RuntimeError("need both provider and location_code to create a hosted router.") url = self.audience+'core/v2/edge-routers' - resource = create_generic_resource(url=url, body=body, headers=headers, proxies=self.proxies, verify=self.verify, wait=wait, sleep=sleep) + resource = create_generic_resource(setup=self, url=url, body=body, wait=wait, sleep=sleep) return(resource) - def create_edge_router_policy(self, name: str, endpoint_attributes: list = [], edge_router_attributes: list = [], wait: int = 30): + def create_edge_router_policy(self, name: str, endpoint_attributes: list = [], edge_router_attributes: list = [], wait: int = 30, sleep: int = 3): """Create an edge router Policy. :param name: a meaningful, unique name @@ -662,9 +662,6 @@ def create_edge_router_policy(self, name: str, endpoint_attributes: list = [], e :param edge_router_attributes: a list of router hashtag role attributes and router name mentions :param wait: seconds to wait for provisioning to finish before raising an exception """ - headers = { - "authorization": "Bearer " + self.token - } for role in endpoint_attributes+edge_router_attributes: if not re.match('^[#@]', role): raise RuntimeError("role attributes on a policy must begin with # or @") @@ -675,7 +672,7 @@ def create_edge_router_policy(self, name: str, endpoint_attributes: list = [], e "edgeRouterAttributes": edge_router_attributes } url = self.audience+'core/v2/edge-router-policies' - resource = create_generic_resource(url=url, body=body, headers=headers, proxies=self.proxies, verify=self.verify, wait=wait) + resource = create_generic_resource(setup=self, url=url, body=body, wait=wait, sleep=sleep) return(resource) @docstring_parameters(valid_service_protocols=VALID_SERVICE_PROTOCOLS) @@ -793,7 +790,7 @@ def create_service_simple(self, name: str, client_host_name: str, client_port: i body['model']['bindEndpointAttributes'] = bind_endpoints url = self.audience+'core/v2/services' - resource = create_generic_resource(url=url, body=body, headers=headers, proxies=self.proxies, verify=self.verify, wait=wait, sleep=sleep) + resource = create_generic_resource(setup=self, url=url, body=body, wait=wait, sleep=sleep) return(resource) # the above method was renamed to follow the development of PSM-based services (platform service models) @@ -833,7 +830,7 @@ def create_service_policy(self, name: str, services: list, endpoints: list, type return(body) url = self.audience+'core/v2/service-policies' - resource = create_generic_resource(url=url, body=body, headers=headers, proxies=self.proxies, verify=self.verify, wait=wait, sleep=sleep) + resource = create_generic_resource(setup=self, url=url, body=body, wait=wait, sleep=sleep) return(resource) def create_service_edge_router_policy(self, name: str, services: list, edge_routers: list, semantic: str = "AnyOf", @@ -864,7 +861,7 @@ def create_service_edge_router_policy(self, name: str, services: list, edge_rout return(body) url = self.audience+'core/v2/service-edge-router-policies' - resource = create_generic_resource(url=url, body=body, headers=headers, proxies=self.proxies, verify=self.verify, wait=wait, sleep=sleep) + resource = create_generic_resource(setup=self, url=url, body=body, wait=wait, sleep=sleep) return(resource) def create_service_with_configs(self, name: str, intercept_config_data: dict, host_config_data: dict, attributes: list = [], @@ -1171,14 +1168,14 @@ def create_service_advanced(self, name: str, endpoints: list, client_hosts: list return(body) url = self.audience+'core/v2/services' - resource = create_generic_resource(url=url, body=body, headers=headers, proxies=self.proxies, verify=self.verify, wait=wait, sleep=sleep) + resource = create_generic_resource(setup=self, url=url, body=body, wait=wait, sleep=sleep) return(resource) # the above method was renamed to follow the development of PSM-based services (platform service models) create_endpoint_service = create_service_advanced def create_app_wan(self, name: str, endpoint_attributes: list = [], service_attributes: list = [], posture_check_attributes: list = [], - wait: int = 10): + wait: int = 10, sleep: int = 3): """Create an AppWAN. :param name: a meaningful, unique name @@ -1200,7 +1197,7 @@ def create_app_wan(self, name: str, endpoint_attributes: list = [], service_attr "postureCheckAttributes": posture_check_attributes } url = self.audience+'core/v2/app-wans' - resource = create_generic_resource(url=url, body=body, headers=headers, proxies=self.proxies, verify=self.verify, wait=wait) + resource = create_generic_resource(setup=self, url=url, body=body, wait=wait, sleep=sleep) return(resource) def get_network_by_name(self, name: str): @@ -1302,7 +1299,7 @@ def wait_for_property_defined(self, property_name: str, property_type: object = while time.time() < now+wait: try: - entity = self.get_resource_by_id(type=entity_type, id=id) + entity = self.get_resource_by_id(type=entity_type, id=id, use_cache=False) except Exception as e: raise RuntimeError(f"problem getting resource by id to get status, caught {e}") @@ -1347,7 +1344,7 @@ def wait_for_entity_name_exists(self, entity_name: str, entity_type: str, wait: found_entities = [] while time.time() < now+wait: try: - found_entities = self.find_resources(type=plural(entity_type), name=entity_name) + found_entities = self.find_resources(type=plural(entity_type), name=entity_name, use_cache=False) except Exception as e: raise RuntimeError(f"error finding resources, caught {e}") @@ -1475,8 +1472,7 @@ def get_resource_status(self, type: str, id: str = None): else: entity_url += f"{plural(type)}/{id}" - headers = {"authorization": "Bearer " + self.token} - resource, status_symbol = get_generic_resource_by_url(setup=self, url=entity_url) + resource, status_symbol = get_generic_resource_by_url(setup=self, url=entity_url, use_cache=False) if resource.get(RESOURCES[plural(type)].status): status = resource[RESOURCES[plural(type)].status] @@ -1588,16 +1584,16 @@ class Networks: networks. """ - def __init__(self, Organization: object) -> None: + def __init__(self, setup: object) -> None: """ - :organization: an instance of netfoundry.Organization provides a session and configuration + :organization: an instance of an object such as netfoundry.Organization provides a session and configuration """ - self.token = Organization.token - self.proxies = Organization.proxies - self.verify = Organization.verify - self.audience = Organization.audience - self.environment = Organization.environment - self.logger = Organization.logger + self.token = setup.token + self.proxies = setup.proxies + self.verify = setup.verify + self.audience = setup.audience + self.environment = setup.environment + self.logger = setup.logger def find_network_domain_resources(self, resource_type: str, **kwargs): """ diff --git a/netfoundry/utility.py b/netfoundry/utility.py index 188448d..d1f4765 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -257,9 +257,9 @@ def get_user_config_dir(): return user_config_path(appname='netfoundry') -def get_generic_resource_by_type_and_id(setup: object, resource_type: str, resource_id: str, accept: str = None, **kwargs): +def get_generic_resource_by_type_and_id(setup: object, resource_type: str, resource_id: str, accept: str = None, use_cache: bool = True, **kwargs): url = f"{setup.audience}{RESOURCES[resource_type].find_url}/{resource_id}" - resource, status_symbol = get_generic_resource_by_url(setup=setup, url=url, accept=accept, **kwargs) + resource, status_symbol = get_generic_resource_by_url(setup=setup, url=url, accept=accept, use_cache=use_cache, **kwargs) return resource, status_symbol @@ -283,7 +283,7 @@ def wait_for_execution(setup: object, url: str, wait: int = 300, sleep: int = 3) status = 'NEW' # time.sleep(sleep) # allow minimal time for the resource status to become available while time.time() < now+wait and status not in expected_statuses: - execution, status_symbol = get_generic_resource_by_url(setup=setup, url=url) + execution, status_symbol = get_generic_resource_by_url(setup=setup, url=url, use_cache=False) if execution.get('status'): # attribute is not None if HTTP OK status = execution['status'] setup.logger.debug(f"{execution['name']} has status {execution['status']}") @@ -333,7 +333,7 @@ def create_generic_resource(setup: object, url: str, body: dict, headers: dict = return resource -def get_generic_resource_by_url(setup: object, url: str, headers: dict = dict(), accept: str = None, **kwargs): +def get_generic_resource_by_url(setup: object, url: str, headers: dict = dict(), accept: str = None, use_cache: bool = True, **kwargs): """ Get, deserialize, and return a single resource. @@ -363,7 +363,11 @@ def get_generic_resource_by_url(setup: object, url: str, headers: dict = dict(), params['beta'] = str() else: setup.logger.debug(f"no handlers specified for url '{url}'") - response = http_cache.get( + if use_cache: + http_session = http_cache + else: + http_session = http + response = http_session.get( url, headers=headers, params=params, @@ -401,7 +405,7 @@ def get_generic_resource_by_url(setup: object, url: str, headers: dict = dict(), get_generic_resource = get_generic_resource_by_type_and_id -def find_generic_resources(setup: object, url: str, headers: dict = dict(), embedded: str = None, accept: str = None, **kwargs): +def find_generic_resources(setup: object, url: str, headers: dict = dict(), embedded: str = None, accept: str = None, use_cache: bool = True, **kwargs): """ Generate each page of a type of resource. @@ -418,8 +422,8 @@ def find_generic_resources(setup: object, url: str, headers: dict = dict(), embe # validate and store the resource type resource_type = get_resource_type_by_url(url) setup.logger.debug(f"detected URL for resource type {resource_type.name}") - if not resource_type.name == "download-urls": - headers['Authorization'] = f"Bearer {setup.token}" + # if not resource_type.name == "download-urls": + headers['Authorization'] = f"Bearer {setup.token}" if HOSTABLE_NET_RESOURCES.get(resource_type.name): params['embed'] = "host" elif resource_type.name in ["process-executions"]: @@ -436,15 +440,15 @@ def find_generic_resources(setup: object, url: str, headers: dict = dict(), embe # normalize output with a default sort param if not params.get('sort'): params["sort"] = "name,asc" - # workaround sort param bugs in MOP-18018, MOP-17863, MOP-18178 - if resource_type.name in ['identities', 'user-identities', 'api-account-identities', 'hosts', 'terminators']: + # workaround sort param bugs in MOP-18018, MOP-17863, MOP-18178, MOP-18366 + if resource_type.name in ['identities', 'user-identities', 'api-account-identities', 'hosts', 'terminators', 'network-versions']: del params['sort'] # only get one page of the requested size, else default page size and all pages if params.get('size'): get_all_pages = False else: - if resource_type.name in ['data-centers', 'roles']: + if resource_type.name in ['roles']: params['size'] = 3000 # workaround last page bug in MOP-17993 else: params['size'] = DEFAULT_PAGE_SIZE @@ -463,8 +467,11 @@ def find_generic_resources(setup: object, url: str, headers: dict = dict(), embe embedded = accept + embedded[0].upper() + embedded[1:] # compose "createEndpointList" from "endpointList" else: setup.logger.warn("ignoring invalid value for header 'accept': '{:s}'".format(accept)) - - response = http_cache.get( + if use_cache: + http_session = http_cache + else: + http_session = http + response = http_session.get( url, headers=headers, params=params, @@ -915,7 +922,7 @@ def send(self, request, **kwargs): http = Session() # no cache -HTTP_CACHE_EXPIRE = 111 +HTTP_CACHE_EXPIRE = 33 http_cache = CachedSession(cache_name=f"{get_user_cache_dir()}/http_cache", backend='sqlite', expire_after=HTTP_CACHE_EXPIRE) # Mount it for both http and https usage adapter = TimeoutHTTPAdapter(timeout=DEFAULT_TIMEOUT, max_retries=RETRY_STRATEGY) From b2fc20f2a9bdd348912dba065917759feed04731 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 21 Jun 2022 14:59:17 -0400 Subject: [PATCH 06/41] require importlib_metadata --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 383d41f..162b4cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,7 @@ requires = [ "setuptools >= 61", "wheel >= 0.37", - "versioneer >= 0.22" + "versioneer >= 0.22", + "importlib_metadata >= 4.2" ] From 509e492f442b0dbfd24c46cb362e3630fce1f9d2 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 21 Jun 2022 15:01:30 -0400 Subject: [PATCH 07/41] finish adapting nfctl to new logging scheme --- netfoundry/ctl.py | 47 +++++++++++++++++++++++++++++++++---------- netfoundry/utility.py | 10 ++++++--- 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index 3a72af2..0211651 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -11,6 +11,7 @@ import platform import re import signal +import jwt import tempfile from builtins import list as blist from json import dumps as json_dumps @@ -40,7 +41,7 @@ from .network import Network, Networks from .network_group import NetworkGroup from .organization import Organization -from .utility import DC_PROVIDERS, EMBED_NET_RESOURCES, IDENTITY_ID_PROPERTIES, MUTABLE_NET_RESOURCES, MUTABLE_RESOURCE_ABBREV, RESOURCE_ABBREV, RESOURCES, any_in, get_generic_resource_by_type_and_id, plural, propid2type, singular +from .utility import DC_PROVIDERS, EMBED_NET_RESOURCES, IDENTITY_ID_PROPERTIES, MUTABLE_NET_RESOURCES, MUTABLE_RESOURCE_ABBREV, RESOURCE_ABBREV, RESOURCES, any_in, get_generic_resource_by_type_and_id, normalize_caseless, plural, propid2type, singular set_metadata(version=f"v{netfoundry_version}", author="NetFoundry", name="nfctl") # must precend import milc.cli from milc import cli, questions # this uses metadata set above @@ -603,7 +604,6 @@ def get(cli, echo: bool = True, spinner: object = None): @cli.argument('-m', '--my-roles', arg_only=True, action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fstore_true", help="filter roles by caller identity") @cli.argument('-a', '--as', dest='accept', arg_only=True, choices=['create'], help="request the as=create alternative form of the resources") @cli.argument('-n', '--names', default=False, action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fstore_boolean", help=argparse.SUPPRESS) -@cli.argument('-n', '--names', default=False, action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fstore_boolean", help=argparse.SUPPRESS) @cli.argument('resource_type', arg_only=True, help='type of resource', metavar="RESOURCE_TYPE", choices=[choice for group in [[type, RESOURCES[type].abbreviation] for type in RESOURCES.keys()] for choice in group]) @cli.subcommand(description='find a collection of resources by type and query') @@ -688,7 +688,6 @@ def list(cli, echo: bool = True, spinner: object = None): for match in matches: valid_keys = valid_keys.union(match.keys()) - valid_keys = valid_keys.intersection(cli.args.keys) # intersection of the set of valid, observed keys in the first match default_keys = ['name', 'label', 'organizationShortName', 'type', 'description', 'edgeRouterAttributes', 'serviceAttributes', 'endpointAttributes', @@ -697,12 +696,11 @@ def list(cli, echo: bool = True, spinner: object = None): 'address', 'binding', 'component'] if cli.config.list.names: # include identity IDs if --names default_keys.extend(IDENTITY_ID_PROPERTIES) - valid_keys = valid_keys.intersection(default_keys) - 'active', 'default', 'region', 'size', 'attributes', 'email', 'productVersion', - 'address', 'binding', 'component'] + if cli.args.keys: + valid_keys = valid_keys.intersection(cli.args.keys) + else: + valid_keys = valid_keys.intersection(default_keys) cli.log.debug(f"filtering matches for valid keys: {str(valid_keys)}") - default_keys.extend(IDENTITY_ID_PROPERTIES) - valid_keys = valid_keys.intersection(default_keys) if valid_keys: cli.log.debug(f"filtering matches for valid keys: {str(valid_keys)}") @@ -732,7 +730,7 @@ def list(cli, echo: bool = True, spinner: object = None): cli.log.debug(f"unexpected value for {k} = {v}") continue # get the resource with the name we're after - resource, status = get_generic_resource_by_type_and_id(org=organization, resource_type=type_by_prop[k], resource_id=v) + resource, status = get_generic_resource_by_type_and_id(setup=organization, resource_type=type_by_prop[k], resource_id=v) if resource.get('name'): # if the name property isn't empty match[k] = f"{resource['name']}" # wedge the name into the ID column @@ -759,7 +757,7 @@ def list(cli, echo: bool = True, spinner: object = None): cli.log.debug(f"unexpected value for {k} = {v}") continue # get the resource with the name we're after - resource, status = get_generic_resource_by_type_and_id(org=organization, resource_type=type_by_prop[k], resource_id=v) + resource, status = get_generic_resource_by_type_and_id(setup=organization, resource_type=type_by_prop[k], resource_id=v) if resource.get('name'): # if the name property isn't empty match[k] = f"{resource['name']}" # wedge the name into the ID column @@ -1215,7 +1213,7 @@ def use_organization(cli, spinner: object = None, prompt: bool = True): raise NFAPINoCredentials() spinner.succeed(f"Logged in profile '{cli.config.general.profile}'") cli.log.debug(f"logged-in organization label is {organization.label}.") - networks = Networks(Organization=organization) + networks = Networks(setup=organization) return organization, networks @@ -1396,6 +1394,33 @@ def get_spinner(cli, text): return inner_spinner +def jwt_decode(token): + # TODO: figure out how to stop doing this because the token is for the + # API, not this app, and so may change algorithm unexpectedly or stop + # being a JWT altogether, currently needed to build the URL for HTTP + # requests, might need to start using env config + """Parse the token and return claimset.""" + try: + claim = jwt.decode(jwt=token, algorithms=["RS256"], options={"verify_signature": False}) + except jwt.exceptions.PyJWTError as e: + raise jwt.exceptions.PyJWTError(f"failed to parse bearer token as JWT, caught {e}") + except Exception as e: + raise RuntimeError(f"unexpected error parsing JWT, caught {e}") + return claim + + +def is_jwt(token): + """If is a JWT then True.""" + try: + jwt_decode(token) + except jwt.exceptions.PyJWTError: + return False + except Exception as e: + raise RuntimeError(f"unexpected error parsing JWT, caught {e}") + else: + return True + + yaml_lexer = get_lexer_by_name("yaml", stripall=True) json_lexer = get_lexer_by_name("json", stripall=True) bash_lexer = get_lexer_by_name("bash", stripall=True) diff --git a/netfoundry/utility.py b/netfoundry/utility.py index d1f4765..f8e0b22 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -1,6 +1,7 @@ """Shared helper functions, constants, and classes.""" import json +import logging import os import re # regex import time # enforce a timeout; sleep @@ -20,14 +21,17 @@ from requests.adapters import HTTPAdapter from requests.exceptions import HTTPError from requests_cache import CachedSession +# FIXME: disable warning for debug proxy +from urllib3 import disable_warnings +from urllib3.exceptions import InsecureRequestWarning from urllib3.util.retry import Retry from .exceptions import UnknownResourceType -# FIXME: disable warning for debug proxy -from urllib3 import disable_warnings -from urllib3.exceptions import InsecureRequestWarning disable_warnings(InsecureRequestWarning) +for name, logger in logging.root.manager.loggerDict.items(): + if name.startswith('requests_cache'): + logger.disabled = True def any_in(a, b): From 93d98b2b57cedb9daeb76f459f3e00aafa0874d9 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 21 Jun 2022 20:39:58 -0400 Subject: [PATCH 08/41] finish adopting the regions endpoint --- netfoundry/ctl.py | 39 ++++++++++------------ netfoundry/network.py | 65 ++++++++++++++++++++----------------- netfoundry/network_group.py | 18 ++++++---- netfoundry/utility.py | 2 +- 4 files changed, 65 insertions(+), 59 deletions(-) diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index 0211651..5e6d96b 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -487,6 +487,14 @@ def get(cli, echo: bool = True, spinner: object = None): matches = organization.find_roles(**cli.args.query) if len(matches) == 1: match = matches[0] + elif cli.args.resource_type == "region": + if 'id' in query_keys: + cli.log.error("regions do not have an ID property, try provider and location_code params") + sysexit(1) + else: + matches = networks.find_regions(**cli.args.query) + if len(matches) == 1: + match = matches[0] elif cli.args.resource_type == "network": if 'id' in query_keys: if len(query_keys) > 1: @@ -524,27 +532,15 @@ def get(cli, echo: bool = True, spinner: object = None): else: cli.log.error("need --network=ACMENet") sysexit(1) - if cli.args.resource_type == "region": - if 'id' in query_keys: - cli.log.warn("regions fetched by ID may not support this network's product version, try provider or locationCode params for safety") - if len(query_keys) > 1: - query_keys.remove('id') - cli.log.warn(f"using 'id' only, ignoring params: '{', '.join(query_keys)}'") - match = network.get_region_by_id(id=cli.args.query['id']) - else: - matches = networks.find_regions(**cli.args.query) - if len(matches) == 1: - match = network.get_region_by_id(id=matches[0]['id']) + if 'id' in query_keys: + if len(query_keys) > 1: + query_keys.remove('id') + cli.log.warn(f"using 'id' only, ignoring params: '{', '.join(query_keys)}'") + match = network.get_resource_by_id(type=cli.args.resource_type, id=cli.args.query['id'], accept=cli.args.accept) else: - if 'id' in query_keys: - if len(query_keys) > 1: - query_keys.remove('id') - cli.log.warn(f"using 'id' only, ignoring params: '{', '.join(query_keys)}'") - match = network.get_resource_by_id(type=cli.args.resource_type, id=cli.args.query['id'], accept=cli.args.accept) - else: - matches = network.find_resources(type=cli.args.resource_type, accept=cli.args.accept, params=cli.args.query) - if len(matches) == 1: - match = matches[0] + matches = network.find_resources(type=cli.args.resource_type, accept=cli.args.accept, params=cli.args.query) + if len(matches) == 1: + match = matches[0] if match: cli.log.debug(f"found exactly one {cli.args.resource_type} by '{', '.join(query_keys)}'") @@ -686,6 +682,7 @@ def list(cli, echo: bool = True, spinner: object = None): valid_keys = set() for match in matches: + # cli.log.debug(match) valid_keys = valid_keys.union(match.keys()) # intersection of the set of valid, observed keys in the first match @@ -973,7 +970,7 @@ def demo(cli): # a list of locations to place a hosted router fabric_placements = [] for region in cli.config.demo.regions: - region_matches = networks.find_regions(provider=cli.config.demo.provider, location_code=region) + region_matches = networks.find_regions(providers=[cli.config.demo.provider], location_code=region) if not len(region_matches) == 1: raise RuntimeError(f"invalid region '{region}'") else: diff --git a/netfoundry/network.py b/netfoundry/network.py index 8f94a7e..9709b08 100644 --- a/netfoundry/network.py +++ b/netfoundry/network.py @@ -267,16 +267,6 @@ def validate_entity_roles(self, entities: list, type: str): return(valid_entities) - def get_region_by_id(self, id: str): - """Get data center region by UUIDv4. - - :param id: required UUIDv4 of data center - """ - url = self.audience+'core/v2/regions/'+id - data_center, status_symbol = get_generic_resource_by_url(setup=self, url=url) - return(data_center) - get_data_center_by_id = get_region_by_id - @docstring_parameters(providers=str(DC_PROVIDERS)) def find_regions(self, **kwargs): """Find regions for hosted router placement. @@ -632,15 +622,9 @@ def create_edge_router(self, name: str, attributes: list = list(), link_listener "linkListener": link_listener, "tunnelerEnabled": tunneler_enabled } - if data_center_id: - self.logger.warning('data_center_id is deprecated by provider, location_code. ') - data_center = self.get_data_center_by_id(id=data_center_id) - body['provider'] = data_center['provider'] - body['locationCode'] = data_center['locationCode'] - body['linkListener'] = True - elif provider or location_code: + if provider or location_code: if provider and location_code: - data_centers = self.get_edge_router_data_centers(provider=provider, location_code=location_code) + data_centers = self.find_regions(provider=provider, location_code=location_code) if len(data_centers) == 1: body['provider'] = provider body['locationCode'] = location_code @@ -1642,18 +1626,39 @@ def get_network_domain_resource(self, resource_type: str, id: str, **kwargs): resource = get_generic_resource_by_url(setup=self, url=url, **params) return(resource) - def find_regions(self, **kwargs): - """Find regions.""" - # data centers returns a list of dicts (data center objects) - params = dict() - for param in kwargs.keys(): - params[param] = kwargs[param] + def find_regions(self, providers: list = [], provider: str = None, location_code: str = None): + """ + Find regions. - if params.get('provider') and not params['provider'] in DC_PROVIDERS: - raise RuntimeError(f"unknown cloud provider '{params['provider']}'. Need one of {str(DC_PROVIDERS)}") + Optionally filter by provider, and optionally get exactly one region by sending both a single element for providers and a location_code + + :param providers: optional list of providers from DC_PROVIDERS + :param provider: optional string matching one provider from DC_PROVIDERS + :param location_code: optional string that uniquely identifies a region for some provider + """ + # regions returns a list of dicts (data center objects) + + if provider: + providers.append(provider) + + if providers: + for provider in providers: + if provider not in DC_PROVIDERS: + raise RuntimeError(f"unknown cloud provider '{provider}'. Need one of {str(DC_PROVIDERS)}") url = self.audience+NET_RESOURCES['regions'].find_url - regions = list() - for i in find_generic_resources(setup=self, url=url, embedded=NET_RESOURCES['regions']._embedded, **params): - regions.extend(i) - return(regions) + + if len(providers) == 1 and location_code: + url = f"{url}/{providers[0]}/{location_code}" + region, status = get_generic_resource_by_url(setup=self, url=url) + return([region]) + else: + regions = list() + for i in find_generic_resources(setup=self, url=url, embedded=NET_RESOURCES['regions']._embedded, providers=providers): + regions.extend(i) + + if location_code and len(regions) > 0: + regions = [r for r in regions if r['locationCode'] == location_code] + return(regions) + else: + return(regions) diff --git a/netfoundry/network_group.py b/netfoundry/network_group.py index 98d60cb..e5791c8 100644 --- a/netfoundry/network_group.py +++ b/netfoundry/network_group.py @@ -1,5 +1,6 @@ """Use a network group and find its networks.""" +# from .organization import Organization from .network import Networks from .utility import NET_RESOURCES, STATUS_CODES, caseless_equal, create_generic_resource, find_generic_resources, get_generic_resource_by_url, http, is_uuidv4, normalize_caseless @@ -12,6 +13,7 @@ class NetworkGroup: def __init__(self, Organization: object, network_group_id: str = None, network_group_name: str = None, group: str = None): """Initialize the network group class with a group name or ID.""" + # self.organization = Organization self.logger = Organization.logger self.network_groups = Organization.find_network_groups_by_organization() if (not network_group_id and not network_group_name) and group: @@ -72,9 +74,9 @@ def __init__(self, Organization: object, network_group_id: str = None, network_g def map_region_id_by_location_code(self): """Map all region ids by their location code e.g. us-west-1: id.""" region_map = dict() - for region in Networks.find_regions(provider='OCP') + Networks.find_regions(provider='AWS'): - region_map[region['locationCode']] = region['id'] - # e.g. { us-east-1: 02f0eb51-fb7a-4d2e-8463-32bd9f6fa4d7 } + for region in Networks.find_regions(providers=['OCI', 'AWS']): + region_map[region['provider']-region['locationCode']] = region['name'] + # e.g. { AWS-us-east-1: 02f0eb51-fb7a-4d2e-8463-32bd9f6fa4d7 } return(region_map) nc_data_centers_by_location = map_region_id_by_location_code @@ -147,7 +149,7 @@ def find_latest_network_version(self, network_versions: list = list(), is_active find_latest_product_version = find_latest_network_version - def create_network(self, name: str, network_group_id: str = None, location: str = "us-east-1", version: str = None, size: str = "medium", wait: int = 1200, sleep: int = 10, **kwargs): + def create_network(self, name: str, network_group_id: str = None, location: str = "us-ashburn-1", provider: str = "OCI", version: str = None, size: str = "medium", wait: int = 1200, sleep: int = 10, **kwargs): """ Create a network in this network group. @@ -159,7 +161,7 @@ def create_network(self, name: str, network_group_id: str = None, location: str """ # my_nc_data_centers_by_location = self.nc_data_centers_by_location() # if not my_nc_data_centers_by_location.get(location): - # raise RuntimeError(f"unexpected network location '{location}'. Valid locations include: {', '.join(my_nc_data_centers_by_location.keys())}.") + # raise RuntimeError(f"unexpected network location '{location}'. Valid locations include: {', '.join(my_nc_data_centers_by_location.keys())}.") # map incongruent api keys from kwargs to function params ("name", "size" are congruent) for param, value in kwargs.items(): @@ -178,13 +180,15 @@ def create_network(self, name: str, network_group_id: str = None, location: str else: self.logger.warn(f"ignoring unexpected keyword argument '{param}'") - matching_regions = Networks.find_regions(region=location) + networks = Networks(setup=self) + matching_regions = networks.find_regions(provider=provider, location_code=location) if not len(matching_regions) == 1: raise RuntimeError(f"failed to find exactly one match for requested controller region '{location}'") body = { "name": name.strip('"'), - "locationCode": location, + "provider": provider, + "region": location, "size": size, } diff --git a/netfoundry/utility.py b/netfoundry/utility.py index f8e0b22..5a69ac3 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -885,7 +885,7 @@ def __post_init__(self): } } -DC_PROVIDERS = ["AWS", "AZURE", "GCP", "OCP"] +DC_PROVIDERS = ["AWS", "AZURE", "GCP", "OCP", "OCI", "ALICLOUD", "NETFOUNDRY"] VALID_SERVICE_PROTOCOLS = ["tcp", "udp"] VALID_SEPARATORS = '[:-]' # : or - will match regex pattern From 08bb0255d8af9a52a12189b2a186c6b40cfe0be3 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 21 Jun 2022 20:52:25 -0400 Subject: [PATCH 09/41] don't wait if no execution link --- netfoundry/utility.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/netfoundry/utility.py b/netfoundry/utility.py index 5a69ac3..4a590e0 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -330,10 +330,12 @@ def create_generic_resource(setup: object, url: str, body: dict, headers: dict = response.raise_for_status() resource = response.json() - if wait: + if wait and resource['_links'].get('execution'): execution_url = resource['_links']['execution']['href'] setup.logger.debug(f"waiting for create {resource_type} execution with url {execution_url}") wait_for_execution(setup=setup, url=execution_url, wait=wait, sleep=sleep) + else: + setup.logger.warn(f"not waiting for create {resource_type} execution") return resource From 7ad878ab77961b8313a7342c80e5b0d17db9f148 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 21 Jun 2022 21:03:16 -0400 Subject: [PATCH 10/41] finish adopting the executions refactor --- netfoundry/ctl.py | 2 +- netfoundry/utility.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index 5e6d96b..b6d7afa 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -955,7 +955,7 @@ def demo(cli): name=network_name, size=cli.config.demo.size, version=cli.config.demo.product_version, - wait=0) # FIXME: don't use wait > 0 until process-executions beta is launched, until then poll for status + wait=333) network, network_group = use_network( cli, organization=organization, diff --git a/netfoundry/utility.py b/netfoundry/utility.py index 4a590e0..3f71c82 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -335,7 +335,7 @@ def create_generic_resource(setup: object, url: str, body: dict, headers: dict = setup.logger.debug(f"waiting for create {resource_type} execution with url {execution_url}") wait_for_execution(setup=setup, url=execution_url, wait=wait, sleep=sleep) else: - setup.logger.warn(f"not waiting for create {resource_type} execution") + setup.logger.warn(f"not waiting for create {resource_type['name']} execution") return resource From 5fec177277701c80ef42d039d48b136172481c4f Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 21 Jun 2022 21:32:29 -0400 Subject: [PATCH 11/41] run demo in OCI --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index eb848a5..22d477b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -84,8 +84,8 @@ jobs: general.verbose=yes || true # FIXME: sometimes config command exits with an error nfctl demo \ --size medium \ - --regions us-west-2 us-east-2 \ - --provider AWS + --regions us-ashburn-1 us-phoenix-1 \ + --provider OCI nfctl \ list services nfctl \ From e65cd9c097d870388f2c53d251ef3b44c320f394 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 21 Jun 2022 21:32:52 -0400 Subject: [PATCH 12/41] de-duplicate the list of providers when finding regions --- netfoundry/network.py | 13 +++++++++---- netfoundry/utility.py | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/netfoundry/network.py b/netfoundry/network.py index 9709b08..b58ee77 100644 --- a/netfoundry/network.py +++ b/netfoundry/network.py @@ -1641,20 +1641,25 @@ def find_regions(self, providers: list = [], provider: str = None, location_code if provider: providers.append(provider) + unique_providers = set() if providers: for provider in providers: - if provider not in DC_PROVIDERS: + if provider in DC_PROVIDERS: + unique_providers.add(provider) + else: raise RuntimeError(f"unknown cloud provider '{provider}'. Need one of {str(DC_PROVIDERS)}") + url = self.audience+NET_RESOURCES['regions'].find_url - if len(providers) == 1 and location_code: - url = f"{url}/{providers[0]}/{location_code}" + if len(unique_providers) == 1 and location_code: + unique_providers_iterator = iter(unique_providers) + url = f"{url}/{next(unique_providers_iterator, None)}/{location_code}" region, status = get_generic_resource_by_url(setup=self, url=url) return([region]) else: regions = list() - for i in find_generic_resources(setup=self, url=url, embedded=NET_RESOURCES['regions']._embedded, providers=providers): + for i in find_generic_resources(setup=self, url=url, embedded=NET_RESOURCES['regions']._embedded, providers=unique_providers): regions.extend(i) if location_code and len(regions) > 0: diff --git a/netfoundry/utility.py b/netfoundry/utility.py index 3f71c82..c8fa831 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -335,7 +335,7 @@ def create_generic_resource(setup: object, url: str, body: dict, headers: dict = setup.logger.debug(f"waiting for create {resource_type} execution with url {execution_url}") wait_for_execution(setup=setup, url=execution_url, wait=wait, sleep=sleep) else: - setup.logger.warn(f"not waiting for create {resource_type['name']} execution") + setup.logger.warn(f"not waiting for create {resource_type.name} execution") return resource From 902498389e4779aeca700fc1c36094483efc37ab Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Wed, 22 Jun 2022 23:02:34 -0400 Subject: [PATCH 13/41] finish cleaning up the function that provides valid region names --- netfoundry/network_group.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/netfoundry/network_group.py b/netfoundry/network_group.py index e5791c8..29fe9e0 100644 --- a/netfoundry/network_group.py +++ b/netfoundry/network_group.py @@ -71,14 +71,14 @@ def __init__(self, Organization: object, network_group_id: str = None, network_g for net in Organization.get_networks_by_group(network_group_id=self.network_group_id): self.network_ids_by_normal_name[normalize_caseless(net['name'])] = net['id'] - def map_region_id_by_location_code(self): - """Map all region ids by their location code e.g. us-west-1: id.""" + def map_region_name_by_provider_and_location_code(self): + """Map all region providers by their location code.""" region_map = dict() - for region in Networks.find_regions(providers=['OCI', 'AWS']): - region_map[region['provider']-region['locationCode']] = region['name'] - # e.g. { AWS-us-east-1: 02f0eb51-fb7a-4d2e-8463-32bd9f6fa4d7 } + networks = Networks(self) + for region in networks.find_regions(providers=['OCI', 'AWS']): + region_map[region['locationCode']] = region['provider'] return(region_map) - nc_data_centers_by_location = map_region_id_by_location_code + nc_data_centers_by_location = map_region_name_by_provider_and_location_code # resolve network UUIDs by name def network_id_by_normal_name(self, name): From 2196ee34d4c381d1e36c2f17bf5fc8b9b82c77ea Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Wed, 22 Jun 2022 23:06:10 -0400 Subject: [PATCH 14/41] cleanup class Network, use the newer wait for execution function --- netfoundry/network.py | 49 +++++++++++++++---------------------------- 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/netfoundry/network.py b/netfoundry/network.py index b58ee77..9447de5 100644 --- a/netfoundry/network.py +++ b/netfoundry/network.py @@ -8,8 +8,8 @@ from netfoundry.exceptions import NetworkBoundaryViolation, UnknownResourceType -from .utility import (DC_PROVIDERS, MUTABLE_NET_RESOURCES, NET_RESOURCES, RESOURCES, STATUS_CODES, VALID_SEPARATORS, VALID_SERVICE_PROTOCOLS, create_generic_resource, docstring_parameters, find_generic_resources, - get_generic_resource_by_url, http, is_uuidv4, normalize_caseless, plural, singular) +from .utility import (DC_PROVIDERS, MUTABLE_NET_RESOURCES, NET_RESOURCES, RESOURCES, STATUS_CODES, VALID_SEPARATORS, VALID_SERVICE_PROTOCOLS, create_generic_resource, docstring_parameters, find_generic_resources, get_generic_resource_by_type_and_id, + get_generic_resource_by_url, http, is_uuidv4, normalize_caseless, plural, singular, wait_for_execution) class Network: @@ -269,9 +269,14 @@ def validate_entity_roles(self, entities: list, type: str): @docstring_parameters(providers=str(DC_PROVIDERS)) def find_regions(self, **kwargs): - """Find regions for hosted router placement. + """ + Find regions for router placement. + + Optionally filter by provider, and optionally get exactly one region by sending both a single element for providers and a location_code - :param provider: optionally filter by data-center region provider, choices: {providers} + :param providers: optional list of providers from DC_PROVIDERS + :param provider: optional string matching one provider from DC_PROVIDERS + :param location_code: optional string that uniquely identifies a region for some provider """ return self.Networks.find_regions(**kwargs) get_edge_router_data_centers = find_regions @@ -309,7 +314,7 @@ def share_endpoint(self, recipient, endpoint_id): if not response_code == STATUS_CODES.codes['ACCEPTED']: raise RuntimeError(f"got unexpected HTTP code {STATUS_CODES._codes[response_code][0].upper()} ({response_code}) and response {response.text}") - def get_resource_by_id(self, type: str, id: str, accept: str = None, use_cache: bool=True): + def get_resource_by_id(self, type: str, id: str, accept: str = None, use_cache: bool = True): """Return an object describing a resource entity. :param type: required string of the singular of an entity type e.g. network, endpoint, service, edge-router, edge-router-policy, posture-check @@ -552,7 +557,6 @@ def create_resource(self, type: str, post: dict, wait: int = 30, sleep: int = 2, :param post: required dictionary with all properties required by the particular resource type's model """ type = plural(type) - headers = {"authorization": "Bearer " + self.token} body = post body['networkId'] = self.id if body.get('name'): @@ -569,9 +573,6 @@ def create_endpoint(self, name: str, attributes: list = [], session_identity: st :param: session_identity is optional string UUID of the identity in the NF organization for which a concurrent web console session is required to activate this endpoint """ - headers = { - "authorization": "Bearer " + self.token - } for role in attributes: if not role[0:1] == '#': raise RuntimeError("hashtag role attributes on an endpoint must begin with #") @@ -587,11 +588,13 @@ def create_endpoint(self, name: str, attributes: list = [], session_identity: st url = self.audience+'core/v2/endpoints' resource = create_generic_resource(setup=self, url=url, body=body, wait=wait, sleep=sleep) + # get it again without cache to ensure the JWT is defined + resource, status = get_generic_resource_by_type_and_id(setup=self, resource_type='endpoints', resource_id=resource['id'], use_cache=False) self.logger.debug(f"created endpoint {resource['name']}") return(resource) @docstring_parameters(providers=str(DC_PROVIDERS)) - def create_edge_router(self, name: str, attributes: list = list(), link_listener: bool = False, data_center_id: str = None, + def create_edge_router(self, name: str, attributes: list = list(), link_listener: bool = False, tunneler_enabled: bool = False, wait: int = 900, sleep: int = 10, progress: bool = False, provider: str = None, location_code: str = None): """Create an edge router. @@ -603,15 +606,11 @@ def create_edge_router(self, name: str, attributes: list = list(), link_listener :param name: a meaningful, unique name :param attributes: a list of hashtag role attributes :param link_listener: true if router should listen for other routers' transit links on 80/tcp, always true if hosted by NetFoundry - :param data_center_id: (DEPRECATED by provider, location_code) the UUIDv4 of a NetFoundry data center location that can host edge routers :param provider: datacenter provider, choices: {providers} :param location_code: provider-specific string identifying the datacenter location e.g. us-west-1 :param tunneler_enabled: true if the built-in tunneler features should be enabled for hosting or interception or both :param wait: seconds to wait for async create to succeed """ - headers = { - "authorization": "Bearer " + self.token - } for role in attributes: if not role[0:1] == '#': raise RuntimeError("hashtag role attributes on an endpoint must begin with #") @@ -688,7 +687,6 @@ def create_service_simple(self, name: str, client_host_name: str, client_port: i :param: endpoints is optional list of strings of hosting endpoints. Selects endpoint-hosting strategy. :param: encryption_required is optional Boolean. Default is to enable edge-to-edge encryption. """ - headers = {"authorization": "Bearer " + self.token} for role in attributes: if not role[0:1] == '#': raise Exception(f'invalid role "{role}". Must begin with "#"') @@ -764,7 +762,7 @@ def create_service_simple(self, name: str, client_host_name: str, client_port: i else: # else assume is a name and resolve to ID try: - name_lookup = self.get_resources(type="endpoints", name=endpoint)[0] + name_lookup = self.get_resources(type="endpoints", name=endpoint, use_cache=False)[0] endpoint_name = name_lookup['name'] except Exception as e: raise RuntimeError(f'failed to find exactly one hosting endpoint named "{endpoint}", caught {e}') @@ -797,9 +795,6 @@ def create_service_policy(self, name: str, services: list, endpoints: list, type bind_endpoints = self.validate_entity_roles(endpoints, type="endpoints") valid_services = self.validate_entity_roles(services, type="services") valid_postures = self.validate_entity_roles(posture_checks, type="posture-checks") - headers = { - "authorization": "Bearer " + self.token - } body = { "networkId": self.id, "name": name.strip('"'), @@ -830,9 +825,6 @@ def create_service_edge_router_policy(self, name: str, services: list, edge_rout """ valid_services = self.validate_entity_roles(services, type="services") valid_routers = self.validate_entity_roles(edge_routers, type="edge-routers") - headers = { - "authorization": "Bearer " + self.token - } body = { "networkId": self.id, "name": name.strip('"'), @@ -1119,7 +1111,7 @@ def create_service_advanced(self, name: str, endpoints: list, client_hosts: list bind_endpoints.append('@'+endpoint_name) # is an existing endpoint's name resolved from UUID else: try: - name_lookup = self.get_resources(type="endpoints", name=endpoint)[0] + name_lookup = self.get_resources(type="endpoints", name=endpoint, use_cache=False)[0] endpoint_name = name_lookup['name'] except Exception as e: raise Exception('ERROR: Failed to find exactly one hosting endpoint named "{}". Caught exception: {}'.format(endpoint, e)) @@ -1127,9 +1119,6 @@ def create_service_advanced(self, name: str, endpoints: list, client_hosts: list else: bind_endpoints.append('@'+endpoint_name) # is an existing endpoint's name - headers = { - "authorization": "Bearer " + self.token - } body = { "networkId": self.id, "name": name, @@ -1167,9 +1156,6 @@ def create_app_wan(self, name: str, endpoint_attributes: list = [], service_attr :param service_attributes: a list of service hashtag role attributes and service names :param posture_check_attributes: a list of posture hashtag role attributes and posture names """ - headers = { - "authorization": "Bearer " + self.token - } for role in endpoint_attributes+service_attributes+posture_check_attributes: if not re.match('^[#@]', role): raise RuntimeError("ERROR: role attributes on an AppWAN must begin with # or @") @@ -1550,10 +1536,10 @@ def delete_resource(self, type: str, id: str = None, wait: int = 0, sleep: int = if wait: if resource.get('_links') and resource['_links'].get('execution'): execution_url = resource['_links'].get('execution')['href'] - self.wait_for_statuses(expected_statuses=RESOURCES["executions"].status_symbols['complete'], type="executions", id=process_id, wait=wait, sleep=sleep) + wait_for_execution(setup=self, url=execution_url, wait=wait, sleep=sleep) return(True) else: - self.logger.warning("unable to wait for async complete because response did not provide a process execution id") + self.logger.error("unable to wait for async complete because response did not provide an execution link") return(False) else: return(True) @@ -1649,7 +1635,6 @@ def find_regions(self, providers: list = [], provider: str = None, location_code else: raise RuntimeError(f"unknown cloud provider '{provider}'. Need one of {str(DC_PROVIDERS)}") - url = self.audience+NET_RESOURCES['regions'].find_url if len(unique_providers) == 1 and location_code: From c038853e19a51bb191e94c1b1388c7ad209f204c Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Thu, 23 Jun 2022 21:57:24 -0400 Subject: [PATCH 15/41] prune a redundant hunk --- netfoundry/ctl.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index b6d7afa..60a183f 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -709,28 +709,6 @@ def list(cli, echo: bool = True, spinner: object = None): cli.log.debug("not filtering output keys") filtered_matches = matches - if cli.config.list.names: - # map any property names that look like a resource ID to the appropriate resource type so we can look up the name later - type_by_prop = dict() - for key in valid_keys: - if key not in ['zitiId']: - if key in IDENTITY_ID_PROPERTIES: - type_by_prop[key] = 'identities' - elif key.endswith('Id'): - type_by_prop[key] = propid2type(key) - - for match in filtered_matches: # for each match - if any_in(type_by_prop.keys(), match.keys()): # if at least one property points to a resolvable ID (fast) - for k, v in match.items(): # for each key in match (slow) - if type_by_prop.get(k): # if this is the property that points to a resolvable ID - if v is None: - cli.log.debug(f"unexpected value for {k} = {v}") - continue - # get the resource with the name we're after - resource, status = get_generic_resource_by_type_and_id(setup=organization, resource_type=type_by_prop[k], resource_id=v) - if resource.get('name'): # if the name property isn't empty - match[k] = f"{resource['name']}" # wedge the name into the ID column - # if echo=False then return the object and parent objects instead of printing with an output format if not echo: return filtered_matches, organization From f26a5d2780a1db61e36a60a316f53c20420060d8 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Thu, 23 Jun 2022 22:02:23 -0400 Subject: [PATCH 16/41] use the default wait for network create --- netfoundry/ctl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index 60a183f..09cbf4f 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -933,7 +933,7 @@ def demo(cli): name=network_name, size=cli.config.demo.size, version=cli.config.demo.product_version, - wait=333) + ) network, network_group = use_network( cli, organization=organization, From bebd055c274a0d6496f91771eaf4601596a66fbb Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jun 2022 12:50:56 -0400 Subject: [PATCH 17/41] prune comments --- netfoundry/network_group.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/netfoundry/network_group.py b/netfoundry/network_group.py index 29fe9e0..cd16f5a 100644 --- a/netfoundry/network_group.py +++ b/netfoundry/network_group.py @@ -159,9 +159,6 @@ def create_network(self, name: str, network_group_id: str = None, location: str :param version: optional product version string like 7.3.17 :param size: optional network configuration metadata name from /core/v2/network-configs e.g. "medium" """ - # my_nc_data_centers_by_location = self.nc_data_centers_by_location() - # if not my_nc_data_centers_by_location.get(location): - # raise RuntimeError(f"unexpected network location '{location}'. Valid locations include: {', '.join(my_nc_data_centers_by_location.keys())}.") # map incongruent api keys from kwargs to function params ("name", "size" are congruent) for param, value in kwargs.items(): From 9f4dc40de3192eeb3a9eb93819b6a8a870267dd9 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 28 Jun 2022 15:43:50 -0400 Subject: [PATCH 18/41] prune comments --- netfoundry/utility.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/netfoundry/utility.py b/netfoundry/utility.py index c8fa831..daf3f29 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -8,7 +8,6 @@ import unicodedata # case insensitive compare in Utility from dataclasses import dataclass, field from json import JSONDecodeError -# from re import sub from stat import filemode from urllib.parse import urlparse from uuid import UUID # validate UUIDv4 strings @@ -235,18 +234,11 @@ def is_uuidv4(string: str): return True -# def eprint(*args, **kwargs): -# """Adapt legacy function to logging.""" -# logging.debug(*args, **kwargs) - - def get_resource_type_by_url(url: str): """Get the resource type definition from a resource URL.""" url_parts = urlparse(url) url_path = url_parts.path resource_type = re.sub(r'/(core|rest|identity|auth|product-metadata)/v\d+/([^/]+)/?.*', r'\2', url_path) - # if resource_type == "download-urls.json": - # resource_type = "download-urls" if RESOURCES.get(resource_type): return RESOURCES.get(resource_type) else: From 0492e58951486087532891f4a4a84b00e85d6248 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 12 Jul 2022 11:52:26 -0400 Subject: [PATCH 19/41] checkpoint for experimental Gateway Service --- netfoundry/ctl.py | 2 ++ netfoundry/network_group.py | 2 +- netfoundry/organization.py | 15 +++++++++------ netfoundry/utility.py | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index 09cbf4f..0cd6b30 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -1153,6 +1153,7 @@ def use_organization(cli, spinner: object = None, prompt: bool = True): expiry_minimum=0, proxy=cli.config.general.proxy, logger=cli.log, + gateway="gatewayv2", ) except NFAPINoCredentials: if prompt: @@ -1177,6 +1178,7 @@ def use_organization(cli, spinner: object = None, prompt: bool = True): expiry_minimum=0, proxy=cli.config.general.proxy, logger=cli.log, + gateway="gatewayv2", ) except PyJWTError: spinner.fail("Not a valid token") diff --git a/netfoundry/network_group.py b/netfoundry/network_group.py index cd16f5a..5625407 100644 --- a/netfoundry/network_group.py +++ b/netfoundry/network_group.py @@ -149,7 +149,7 @@ def find_latest_network_version(self, network_versions: list = list(), is_active find_latest_product_version = find_latest_network_version - def create_network(self, name: str, network_group_id: str = None, location: str = "us-ashburn-1", provider: str = "OCI", version: str = None, size: str = "medium", wait: int = 1200, sleep: int = 10, **kwargs): + def create_network(self, name: str, network_group_id: str = None, location: str = "eu-amsterdam-1", provider: str = "OCI", version: str = None, size: str = "medium", wait: int = 1200, sleep: int = 10, **kwargs): """ Create a network in this network group. diff --git a/netfoundry/organization.py b/netfoundry/organization.py index 125eaa5..7103c9e 100644 --- a/netfoundry/organization.py +++ b/netfoundry/organization.py @@ -43,8 +43,10 @@ def __init__(self, log_file: str = None, debug: bool = False, logger: logging.Logger = None, - proxy: str = None): + proxy: str = None, + gateway: str = "gateway"): """Initialize an instance of organization.""" + self.gateway = gateway # set debug and file if specified and let the calling application dictate logging handlers self.log_file = log_file self.debug = debug @@ -249,7 +251,8 @@ def __init__(self, self.logger.warning(f"unexpected environment '{self.environment}'") if self.environment and not self.audience: - self.audience = f'https://gateway.{self.environment}.netfoundry.io/' + self.audience = f'https://{self.gateway}.{self.environment}.netfoundry.io/' + self.logger.debug(f"computed audience URL from gateway and environment: {self.audience}") if self.environment and self.audience: if not re.search(self.environment, self.audience): @@ -284,15 +287,15 @@ def __init__(self, # extract the environment name from the authorization URL aka token API endpoint if self.environment is None: self.environment = re.sub(r'https://netfoundry-([^-]+)-.*', r'\1', token_endpoint, re.IGNORECASE) - self.logger.debug(f"using environment parsed from token_endpoint URL {self.environment}") + self.logger.debug(f"using environment parsed from authenticationUrl: {self.environment}") # re: scope: we're not using scopes with Cognito, but a non-empty value is required; # hence "/ignore-scope" - scope = "https://gateway."+self.environment+".netfoundry.io//ignore-scope" + scope = f"https://{self.gateway}.{self.environment}.netfoundry.io//ignore-scope" + self.logger.debug(f"computed scope URL from gateway and environment: {scope}") # we can gather the URL of the API from the first part of the scope string by # dropping the scope suffix self.audience = scope.replace(r'/ignore-scope', '') - self.logger.debug(f"using audience parsed from token_endpoint URL {self.audience}") - # e.g. https://gateway.production.netfoundry.io/ + self.logger.debug(f"using audience parsed from authenticationUrl: {self.audience}") assertion = { "scope": scope, "grant_type": "client_credentials" diff --git a/netfoundry/utility.py b/netfoundry/utility.py index daf3f29..830f92e 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -185,7 +185,7 @@ def jwt_environment(setup: object): else: if re.match(r'https://cognito-', iss): - environment = re.sub(r'https://gateway\.([^.]+)\.netfoundry\.io.*', r'\1', claim['scope']) + environment = re.sub(f'https://{setup.gateway}\.([^.]+)\.netfoundry\.io.*', r'\1', claim['scope']) setup.logger.debug(f"matched Cognito issuer URL convention, found environment '{environment}'") elif re.match(r'.*\.auth0\.com', iss): environment = re.sub(r'https://netfoundry-([^.]+)\.auth0\.com.*', r'\1', claim['iss']) From 1683c3b189051bf93f802bc37d339ab6edafc6a7 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Wed, 27 Jul 2022 10:58:08 -0400 Subject: [PATCH 20/41] make the Gateway Service domain name configurable --- netfoundry/ctl.py | 5 +++-- netfoundry/organization.py | 15 +++++++++++---- netfoundry/utility.py | 4 +++- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index 0cd6b30..ec399e3 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -94,6 +94,7 @@ def __call__(self, parser, namespace, values, option_string=None): @cli.argument('-Y', '--yes', action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fstore_true", arg_only=True, help='answer yes to potentially-destructive operations') @cli.argument('-W', '--wait', help='seconds to wait for long-running processes to finish', default=900) @cli.argument('--proxy', help=argparse.SUPPRESS) +@cli.argument('--gateway', default="gateway", help=argparse.SUPPRESS) @cli.entrypoint('configure the CLI to manage a network') def main(cli): """Configure the CLI to manage a network.""" @@ -1153,7 +1154,7 @@ def use_organization(cli, spinner: object = None, prompt: bool = True): expiry_minimum=0, proxy=cli.config.general.proxy, logger=cli.log, - gateway="gatewayv2", + gateway=cli.config.general.gateway, ) except NFAPINoCredentials: if prompt: @@ -1178,7 +1179,7 @@ def use_organization(cli, spinner: object = None, prompt: bool = True): expiry_minimum=0, proxy=cli.config.general.proxy, logger=cli.log, - gateway="gatewayv2", + gateway=cli.config.general.gateway, ) except PyJWTError: spinner.fail("Not a valid token") diff --git a/netfoundry/organization.py b/netfoundry/organization.py index 7103c9e..5cd5506 100644 --- a/netfoundry/organization.py +++ b/netfoundry/organization.py @@ -46,7 +46,6 @@ def __init__(self, proxy: str = None, gateway: str = "gateway"): """Initialize an instance of organization.""" - self.gateway = gateway # set debug and file if specified and let the calling application dictate logging handlers self.log_file = log_file self.debug = debug @@ -81,6 +80,9 @@ def __init__(self, else: self.verify = False + self.gateway = gateway + self.logger.debug(f"got 'gateway' param {self.gateway}") + epoch = round(time.time()) self.expiry_seconds = 0 # initialize a placeholder for remaining seconds until expiry client_id = None @@ -258,6 +260,8 @@ def __init__(self, if not re.search(self.environment, self.audience): self.logger.error(f"mismatched audience URL '{self.audience}' and environment '{self.environment}'") exit(1) + else: + self.logger.debug(f"found audience already computed '{self.audience}' and matching environment '{self.environment}'") # the purpose of this try-except block is to soft-fail all attempts # to parse the JWT, which is intended for the API, not this @@ -290,12 +294,15 @@ def __init__(self, self.logger.debug(f"using environment parsed from authenticationUrl: {self.environment}") # re: scope: we're not using scopes with Cognito, but a non-empty value is required; # hence "/ignore-scope" - scope = f"https://{self.gateway}.{self.environment}.netfoundry.io//ignore-scope" - self.logger.debug(f"computed scope URL from gateway and environment: {scope}") + scope = f"https://gateway.{self.environment}.netfoundry.io//ignore-scope" + self.logger.debug(f"computed scope URL from 'gateway' and environment: {scope}") # we can gather the URL of the API from the first part of the scope string by # dropping the scope suffix self.audience = scope.replace(r'/ignore-scope', '') - self.logger.debug(f"using audience parsed from authenticationUrl: {self.audience}") + self.logger.debug(f"computed audience from authenticationUrl sans the trailing '/ignore-scope': {self.audience}") + audience_parts = self.audience.split('.') + self.audience = '.'.join([f"https://{self.gateway}"]+audience_parts[1:]) + self.logger.debug(f"computed audience with substituted param 'gateway': {self.audience}") assertion = { "scope": scope, "grant_type": "client_credentials" diff --git a/netfoundry/utility.py b/netfoundry/utility.py index 830f92e..704952c 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -185,7 +185,7 @@ def jwt_environment(setup: object): else: if re.match(r'https://cognito-', iss): - environment = re.sub(f'https://{setup.gateway}\.([^.]+)\.netfoundry\.io.*', r'\1', claim['scope']) + environment = re.sub(r'https://gateway\.([^.]+)\.netfoundry\.io.*', r'\1', claim['scope']) setup.logger.debug(f"matched Cognito issuer URL convention, found environment '{environment}'") elif re.match(r'.*\.auth0\.com', iss): environment = re.sub(r'https://netfoundry-([^.]+)\.auth0\.com.*', r'\1', claim['iss']) @@ -319,6 +319,8 @@ def create_generic_resource(setup: object, url: str, body: dict, headers: dict = proxies=setup.proxies, verify=setup.verify, ) + if response.status_code in range(400, 600): + setup.logger.debug(response.request) response.raise_for_status() resource = response.json() From eba3964e01d94fc411a98a0c2a1823a4719caecd Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Fri, 29 Jul 2022 10:25:51 -0400 Subject: [PATCH 21/41] finish cleaning up references to old network-groups API endpoint --- netfoundry/ctl.py | 6 ++---- netfoundry/network_group.py | 8 ++++---- netfoundry/organization.py | 11 ++++++++--- netfoundry/utility.py | 17 +++++++++++------ 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index ec399e3..7aaf42d 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -156,7 +156,7 @@ def login(cli): summary_table = [] summary_table.append(['Caller ID', f"{summary_object['caller']['name']} ({summary_object['caller']['email']}) in {organization.label} ({organization.name})"]) if network_group: - summary_table.append(['Network Resource Group', f"{summary_object['network_group']['name']} ({summary_object['network_group']['organizationShortName']}) with {summary_object['network_group']['networks_count']} networks"]) + summary_table.append(['Network Resource Group', f"{summary_object['network_group']['name']} ({summary_object['network_group']['shortName']}) with {summary_object['network_group']['networks_count']} networks"]) if network: summary_table.append(['Configured Network', f"{summary_object['network']['name']} - {summary_object['network']['productVersion']} - {summary_object['network']['status']}"]) if cli.config.general.borders: @@ -687,13 +687,11 @@ def list(cli, echo: bool = True, spinner: object = None): valid_keys = valid_keys.union(match.keys()) # intersection of the set of valid, observed keys in the first match - default_keys = ['name', 'label', 'organizationShortName', 'type', 'description', + default_keys = ['name', 'label', 'shortName', 'type', 'description', 'edgeRouterAttributes', 'serviceAttributes', 'endpointAttributes', 'status', 'zitiId', 'provider', 'locationCode', 'ipAddress', 'networkVersion', 'active', 'default', 'region', 'size', 'attributes', 'email', 'productVersion', 'address', 'binding', 'component'] - if cli.config.list.names: # include identity IDs if --names - default_keys.extend(IDENTITY_ID_PROPERTIES) if cli.args.keys: valid_keys = valid_keys.intersection(cli.args.keys) else: diff --git a/netfoundry/network_group.py b/netfoundry/network_group.py index 5625407..a373322 100644 --- a/netfoundry/network_group.py +++ b/netfoundry/network_group.py @@ -28,14 +28,14 @@ def __init__(self, Organization: object, network_group_id: str = None, network_g self.network_group_id = network_group_id network_group_matches = [ng for ng in self.network_groups if ng['id'] == network_group_id] if len(network_group_matches) == 1: - self.network_group_name = network_group_matches[0]['organizationShortName'] + self.network_group_name = network_group_matches[0]['shortName'] self.logger.debug(f"found one match for group id '{network_group_id}'") else: raise RuntimeError(f"there was not exactly one network group matching the id '{network_group_id}'") # TODO: review the use of org short name ref https://mattermost.tools.netfoundry.io/netfoundry/pl/gegyzuybypb9jxnrw1g1imjywh elif network_group_name: self.network_group_name = network_group_name - network_group_matches = [ng for ng in self.network_groups if caseless_equal(ng['organizationShortName'], self.network_group_name)] + network_group_matches = [ng for ng in self.network_groups if caseless_equal(ng['shortName'], self.network_group_name)] if len(network_group_matches) == 1: self.network_group_id = network_group_matches[0]['id'] self.logger.debug(f"found one match for group short name '{network_group_name}'") @@ -44,10 +44,10 @@ def __init__(self, Organization: object, network_group_id: str = None, network_g elif len(self.network_groups) > 0: # first network group is typically the only network group self.network_group_id = self.network_groups[0]['id'] - self.network_group_name = normalize_caseless(self.network_groups[0]['organizationShortName']) + self.network_group_name = normalize_caseless(self.network_groups[0]['shortName']) # warn if there are other groups if len(self.network_groups) > 1: - self.logger.warning(f"using first network group {self.network_group_name} and ignoring {len(self.network_groups) - 1} other(s) e.g. {self.network_groups[1]['organizationShortName']}, etc...") + self.logger.warning(f"using first network group {self.network_group_name} and ignoring {len(self.network_groups) - 1} other(s) e.g. {self.network_groups[1]['shortName']}, etc...") elif len(self.network_groups) == 1: self.logger.debug(f"using the only available network group: {self.network_group_name}") else: diff --git a/netfoundry/organization.py b/netfoundry/organization.py index 5cd5506..ec93d88 100644 --- a/netfoundry/organization.py +++ b/netfoundry/organization.py @@ -80,7 +80,12 @@ def __init__(self, else: self.verify = False - self.gateway = gateway + # users of older versions of nfsupport-cli will send literal None until they upgrade to a version that provides the --gateway option + if gateway is None: + self.gateway = "gateway" + else: + self.gateway = gateway + self.logger.debug(f"got 'gateway' param {self.gateway}") epoch = round(time.time()) @@ -554,7 +559,7 @@ def get_network_group(self, network_group_id): :param network_group_id: the UUID of the network group """ - url = self.audience+'rest/v1/network-groups/'+network_group_id + url = self.audience+'core/v2/network-groups/'+network_group_id try: network_group, status_symbol = get_generic_resource_by_url(setup=self, url=url) except Exception as e: @@ -595,7 +600,7 @@ def find_network_groups_by_organization(self, **kwargs): :param str kwargs: filter results by any supported query param """ - url = self.audience+'rest/v1/network-groups' + url = self.audience+'core/v2/network-groups' network_groups = list() for i in find_generic_resources(setup=self, url=url, embedded=RESOURCES['network-groups']._embedded, **kwargs): network_groups.extend(i) diff --git a/netfoundry/utility.py b/netfoundry/utility.py index 704952c..eb5855d 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -319,8 +319,15 @@ def create_generic_resource(setup: object, url: str, body: dict, headers: dict = proxies=setup.proxies, verify=setup.verify, ) - if response.status_code in range(400, 600): - setup.logger.debug(response.request) + if response.status_code in range(200, 600): + req = response.request + setup.logger.debug( + '%s\n%s\r\n%s\r\n\r\n%s', + '-----------START-----------', + req.method + ' ' + req.url, + '\r\n'.join('{}: {}'.format(k, v) for k, v in req.headers.items()), + req.body + ) response.raise_for_status() resource = response.json() @@ -456,8 +463,6 @@ def find_generic_resources(setup: object, url: str, headers: dict = dict(), embe # only get requested page, else first page and all pages if params.get('page'): get_all_pages = False - elif resource_type.name == 'network-groups': - params['page'] = 1 # start at 1 instead of 0 to workaround https://netfoundry.atlassian.net/browse/MOP-17890 else: params['page'] = 0 @@ -660,6 +665,8 @@ def __post_init__(self): if self.find_url == 'default': if self.domain == 'network': setattr(self, 'find_url', f'core/v2/{self.name}') + elif self.domain == 'network-group': + setattr(self, 'find_url', f'core/v2/{self.name}') elif self.domain == 'identity': setattr(self, 'find_url', f'identity/v1/{self.name}') elif self.domain == 'authorization': @@ -717,10 +724,8 @@ def __post_init__(self): 'network-groups': ResourceType( name='network-groups', domain='network-group', - _embedded='organizations', mutable=False, embeddable=False, - find_url="iframe.php?url=https%3A%2F%2Fgithub.com%2Frest%2Fv1%2Fnetwork-groups", ), 'networks': ResourceType( name='networks', From 79f96aae58dedb28e8d246dca24d0a2d0674709d Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Fri, 29 Jul 2022 10:26:55 -0400 Subject: [PATCH 22/41] stop testing the debug message for creating a resource --- netfoundry/utility.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netfoundry/utility.py b/netfoundry/utility.py index eb5855d..78ab4a8 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -319,7 +319,7 @@ def create_generic_resource(setup: object, url: str, body: dict, headers: dict = proxies=setup.proxies, verify=setup.verify, ) - if response.status_code in range(200, 600): + if response.status_code in range(400, 600): req = response.request setup.logger.debug( '%s\n%s\r\n%s\r\n\r\n%s', From d09f51eded17a292e45d7a8b487206912083ec0a Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Sun, 19 Oct 2025 12:11:44 -0400 Subject: [PATCH 23/41] configure flake8 --- .python-flake8-config => .flake8 | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .python-flake8-config => .flake8 (100%) diff --git a/.python-flake8-config b/.flake8 similarity index 100% rename from .python-flake8-config rename to .flake8 From 16be19be6ecbd6ffa8aa750aec3c9b13e5e60250 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Sun, 19 Oct 2025 12:11:59 -0400 Subject: [PATCH 24/41] version constrain requests --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 14059a7..151defa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ install_requires = pygments >= 2.11 pyjwt >= 2.3 pyyaml >= 5.4 - requests >= 2.27 + requests >= 2.27, < 2.30 tabulate >= 0.8 requests-cache >= 0.9.4 setup_requires = From 188d9c502c89feab26759967a34d0a388e6078bd Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Sun, 19 Oct 2025 12:12:56 -0400 Subject: [PATCH 25/41] stop importing unused blist --- netfoundry/ctl.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index 7aaf42d..34fe585 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -13,7 +13,6 @@ import signal import jwt import tempfile -from builtins import list as blist from json import dumps as json_dumps from json import load as json_load from json import loads as json_loads @@ -43,9 +42,12 @@ from .organization import Organization from .utility import DC_PROVIDERS, EMBED_NET_RESOURCES, IDENTITY_ID_PROPERTIES, MUTABLE_NET_RESOURCES, MUTABLE_RESOURCE_ABBREV, RESOURCE_ABBREV, RESOURCES, any_in, get_generic_resource_by_type_and_id, normalize_caseless, plural, propid2type, singular -set_metadata(version=f"v{netfoundry_version}", author="NetFoundry", name="nfctl") # must precend import milc.cli -from milc import cli, questions # this uses metadata set above -from milc.subcommand import config # this creates the config subcommand +# must precend import milc.cli +set_metadata(version=f"v{netfoundry_version}", author="NetFoundry", name="nfctl") +# this uses metadata set above +from milc import cli, questions # noqa: E402 +# this creates the config subcommand +from milc.subcommand import config # noqa: F401,E402 if platform.system() == 'Linux': # this allows the app the terminate gracefully when piped to a truncating consumer like `head` From 9b72a4730fbb3271b84eb3978d9963f4b26fb6e4 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Sun, 19 Oct 2025 12:13:11 -0400 Subject: [PATCH 26/41] version constrain nf network product --- netfoundry/network.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/netfoundry/network.py b/netfoundry/network.py index 9447de5..a0f10ff 100644 --- a/netfoundry/network.py +++ b/netfoundry/network.py @@ -4,6 +4,7 @@ import re import time +from packaging.version import parse from requests.exceptions import JSONDecodeError from netfoundry.exceptions import NetworkBoundaryViolation, UnknownResourceType @@ -56,7 +57,7 @@ def __init__(self, NetworkGroup: object, network_id: str = None, network_name: s self.product_version = self.describe['productVersion'] self.owner_identity_id = self.describe['ownerIdentityId'] self.size = self.describe['size'] - self.o365_breakout_category = self.describe['o365BreakoutCategory'] + self.o365_breakout_category = self.describe.get('o365BreakoutCategory') self.created_at = self.describe['createdAt'] self.updated_at = self.describe['updatedAt'] self.created_by = self.describe['createdBy'] @@ -1228,6 +1229,9 @@ def get_controller_session(self, id: str): Note that this function requires privileged access to the controller and is intended for emergency, read-only operations by customer support engineers. :param id: the UUID of the network controller """ + + if parse(self.product_version) >= parse("8.0.0"): + raise RuntimeError(f"get_controller_session() is unavailable in network version {self.product_version} and later.") url = self.audience+'core/v2/network-controllers/'+id+'/session' try: session, status_symbol = get_generic_resource_by_url(setup=self, url=url) From 3f1d8308f651c9f39bd919f9680a4b00309906e6 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Sun, 19 Oct 2025 13:24:58 -0400 Subject: [PATCH 27/41] bump versions --- .github/workflows/main.yml | 22 +++++++++++----------- docker/Dockerfile | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 22d477b..951e950 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,14 +19,14 @@ jobs: build_pypi_and_docker: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v5 with: fetch-depth: 0 # unshallow checkout enables setuptools_scm to infer PyPi version from Git - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v6 with: - python-version: '3.7' + python-version: '3.13' - name: Install dependencies run: | @@ -37,7 +37,7 @@ jobs: run: python -m build - name: Upload Wheel Artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: netfoundry-wheel-${{ github.run_id }} path: dist/netfoundry-*.whl @@ -98,7 +98,7 @@ jobs: delete network - name: Publish Test Package - uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + uses: pypa/gh-action-pypi-publish@v1 with: user: __token__ password: ${{ secrets.TEST_PYPI_API_TOKEN }} @@ -120,21 +120,21 @@ jobs: - name: Publish Release to PyPi if: github.event.action == 'published' - uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + uses: pypa/gh-action-pypi-publish@v1 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} - name: Attach Wheel Artifact to GH Release if: ${{ github.event.action == 'published' }} - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: files: dist/netfoundry-*.whl fail_on_unmatched_files: true generate_release_notes: true - name: Set up QEMU - uses: docker/setup-qemu-action@master + uses: docker/setup-qemu-action@v3 with: platforms: amd64,arm64 # ignore arm/v7 (32bit) because unsupported by "cryptography" dep of @@ -142,16 +142,16 @@ jobs: - name: Set up Docker BuildKit id: buildx - uses: docker/setup-buildx-action@master + uses: docker/setup-buildx-action@v3 - name: Login to Docker Hub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_HUB_API_USER }} password: ${{ secrets.DOCKER_HUB_API_TOKEN }} - name: Build & Push Multi-Platform Container - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v6 with: context: ${{ github.workspace }} # build context is workspace so we can copy artifacts from ./dist/ file: ${{ github.workspace }}/docker/Dockerfile diff --git a/docker/Dockerfile b/docker/Dockerfile index edf66fa..12dc7d5 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim-buster +FROM python:3.13-slim-bookworm COPY ./dist/netfoundry-*.whl /tmp/ RUN pip install --upgrade pip RUN pip install /tmp/netfoundry-*.whl From 46c59249af8ad20af2bf6e0d48803fa84df60808 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Sun, 19 Oct 2025 14:04:13 -0400 Subject: [PATCH 28/41] bump actions --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 951e950..ec77cb5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -98,7 +98,7 @@ jobs: delete network - name: Publish Test Package - uses: pypa/gh-action-pypi-publish@v1 + uses: pypa/gh-action-pypi-publish@v1.13.0 with: user: __token__ password: ${{ secrets.TEST_PYPI_API_TOKEN }} @@ -120,7 +120,7 @@ jobs: - name: Publish Release to PyPi if: github.event.action == 'published' - uses: pypa/gh-action-pypi-publish@v1 + uses: pypa/gh-action-pypi-publish@v1.13.0 with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} From d9b3c0d1407e74a6ada81777b886d70aebc227be Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Sun, 19 Oct 2025 14:05:55 -0400 Subject: [PATCH 29/41] install setuptools --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ec77cb5..5938e20 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,7 +31,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install build + pip install build setuptools - name: Build Package run: python -m build From 6a572826cf7285a19a95d23187b7038d65a23b5f Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Sun, 19 Oct 2025 14:12:18 -0400 Subject: [PATCH 30/41] use py 3.12 --- .github/workflows/main.yml | 7 +++++-- docker/Dockerfile | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5938e20..143c148 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,7 +26,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: '3.13' + python-version: '3.12' - name: Install dependencies run: | @@ -72,12 +72,15 @@ jobs: register-python-argcomplete nfctl - name: Run the NF CLI demo to test installed version + shell: bash env: NETFOUNDRY_CLIENT_ID: ${{ secrets.NETFOUNDRY_CLIENT_ID }} NETFOUNDRY_PASSWORD: ${{ secrets.NETFOUNDRY_PASSWORD }} NETFOUNDRY_OAUTH_URL: ${{ secrets.NETFOUNDRY_OAUTH_URL }} run: | - set -x + set -o xtrace + set -o pipefail + nfctl config \ general.network=$(nfctl demo --echo-name --prefix 'gh-${{ github.run_id }}') \ general.yes=True \ diff --git a/docker/Dockerfile b/docker/Dockerfile index 12dc7d5..6db8e49 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.13-slim-bookworm +FROM python:3.12-slim-bookworm COPY ./dist/netfoundry-*.whl /tmp/ RUN pip install --upgrade pip RUN pip install /tmp/netfoundry-*.whl From 61947c9ebd06b7e33afee5f8976a10f83ee63700 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Sun, 19 Oct 2025 14:19:28 -0400 Subject: [PATCH 31/41] refactor for new milc api --- netfoundry/ctl.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index 34fe585..ddca9c8 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -25,7 +25,7 @@ from xml.sax.xmlreader import InputSource from jwt.exceptions import PyJWTError -from milc import set_metadata # this function needed to set metadata immediately below +# milc metadata will be set after cli import from pygments import highlight from pygments.formatters import Terminal256Formatter from pygments.lexers import get_lexer_by_name, load_lexer_from_file @@ -42,10 +42,10 @@ from .organization import Organization from .utility import DC_PROVIDERS, EMBED_NET_RESOURCES, IDENTITY_ID_PROPERTIES, MUTABLE_NET_RESOURCES, MUTABLE_RESOURCE_ABBREV, RESOURCE_ABBREV, RESOURCES, any_in, get_generic_resource_by_type_and_id, normalize_caseless, plural, propid2type, singular -# must precend import milc.cli -set_metadata(version=f"v{netfoundry_version}", author="NetFoundry", name="nfctl") -# this uses metadata set above +# import milc cli from milc import cli, questions # noqa: E402 +# set milc options using new API +cli.milc_options(name='nfctl', author='NetFoundry', version=f'v{netfoundry_version}') # this creates the config subcommand from milc.subcommand import config # noqa: F401,E402 @@ -871,7 +871,7 @@ def delete(cli): sysexit(1) -@cli.argument("-p", "--prefix", default=f"{cli.prog_name}-demo", help="choose a network name prefix to identify all of your demos") +@cli.argument("-p", "--prefix", default="nfctl-demo", help="choose a network name prefix to identify all of your demos") @cli.argument("-j", "--jwt", action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fstore_boolean", default=True, help="save the one-time enroll token for each demo identity in the current directory") @cli.argument("-e", "--echo-name", arg_only=True, action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fstore_true", default=False, help="only echo a friendly network name then exit") @cli.argument("-s", "--size", default="medium", help=argparse.SUPPRESS) # troubleshoot scale-up instance size factor From 4f50759741f4b096a83eeca3cea890fa6d4e52f3 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Thu, 13 Nov 2025 13:25:15 -0500 Subject: [PATCH 32/41] adapt to breaking change in requests 2.30 --- netfoundry/ctl.py | 1 + netfoundry/network.py | 2 +- netfoundry/network_group.py | 2 +- netfoundry/organization.py | 2 +- netfoundry/utility.py | 10 +++++++--- setup.cfg | 2 +- 6 files changed, 12 insertions(+), 7 deletions(-) diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index ddca9c8..0430149 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -934,6 +934,7 @@ def demo(cli): name=network_name, size=cli.config.demo.size, version=cli.config.demo.product_version, + provider=cli.config.demo.provider, ) network, network_group = use_network( cli, diff --git a/netfoundry/network.py b/netfoundry/network.py index a0f10ff..0018e4e 100644 --- a/netfoundry/network.py +++ b/netfoundry/network.py @@ -5,7 +5,7 @@ import time from packaging.version import parse -from requests.exceptions import JSONDecodeError +from json import JSONDecodeError from netfoundry.exceptions import NetworkBoundaryViolation, UnknownResourceType diff --git a/netfoundry/network_group.py b/netfoundry/network_group.py index a373322..45d332f 100644 --- a/netfoundry/network_group.py +++ b/netfoundry/network_group.py @@ -173,7 +173,7 @@ def create_network(self, name: str, network_group_id: str = None, location: str elif param == 'productVersion': if version: self.logger.debug("clobbering param 'version' with kwarg 'productVersion'") - version == value + version = value else: self.logger.warn(f"ignoring unexpected keyword argument '{param}'") diff --git a/netfoundry/organization.py b/netfoundry/organization.py index ec93d88..492342f 100644 --- a/netfoundry/organization.py +++ b/netfoundry/organization.py @@ -386,7 +386,7 @@ def __init__(self, if self.organizations_by_label.get(organization_label): self.describe = self.get_organization(id=self.organizations_by_label[organization_label]) else: - raise RuntimeError(f"failed to find org label {organization_label} in the list of orgs {', '.join(self.organizations_by_label.keys())}") + raise RuntimeError(f"failed to find org label {organization_label} in the list of {len(self.organizations_by_label)} available organizations") else: self.describe = self.get_organization(id=self.caller['organizationId']) diff --git a/netfoundry/utility.py b/netfoundry/utility.py index 78ab4a8..dbcfa87 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -485,6 +485,12 @@ def find_generic_resources(setup: object, url: str, headers: dict = dict(), embe ) response.raise_for_status() resource_page = response.json() + + # Handle non-paginated endpoints that return direct lists + if isinstance(resource_page, list): + yield resource_page + return + if isinstance(resource_page, dict) and resource_page.get('page'): try: total_pages = resource_page['page']['totalPages'] @@ -515,8 +521,6 @@ def find_generic_resources(setup: object, url: str, headers: dict = dict(), embe # then yield subsequent pages, if applicable if get_all_pages and total_pages > 1: # get_all_pages is False if param 'page' or 'size' to stop recursion and get a single page next_range_lower, next_range_upper = params['page'] + 1, total_pages - if resource_type.name == 'network-groups': - next_range_upper += 1 # network-groups pages are 1-based and so +1 upper limit for next_page in range(next_range_lower, next_range_upper): params['page'] = next_page try: @@ -902,7 +906,7 @@ def decorated(ref): RETRY_STRATEGY = Retry( total=5, status_forcelist=[403, 404, 413, 429, 503], # The API responds 403 and 404 for not-yet-existing executions for some async operations - method_whitelist=["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"], + allowed_methods=["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"], backoff_factor=1 ) DEFAULT_TIMEOUT = 31 # seconds, Gateway Service waits 30s before responding with an error code e.g. 503 and diff --git a/setup.cfg b/setup.cfg index 151defa..14059a7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ install_requires = pygments >= 2.11 pyjwt >= 2.3 pyyaml >= 5.4 - requests >= 2.27, < 2.30 + requests >= 2.27 tabulate >= 0.8 requests-cache >= 0.9.4 setup_requires = From c167a95370aef53c78f70d3fccf2584ded402fb4 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Thu, 13 Nov 2025 13:36:31 -0500 Subject: [PATCH 33/41] fix demo region --- netfoundry/ctl.py | 1 + netfoundry/demo.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index 0430149..38b19be 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -935,6 +935,7 @@ def demo(cli): size=cli.config.demo.size, version=cli.config.demo.product_version, provider=cli.config.demo.provider, + location=cli.config.demo.regions[0], # Use first region for network location ) network, network_group = use_network( cli, diff --git a/netfoundry/demo.py b/netfoundry/demo.py index 35610cb..e8f8903 100755 --- a/netfoundry/demo.py +++ b/netfoundry/demo.py @@ -28,11 +28,13 @@ def importlib_load_entry_point(spec, group, name): globals().setdefault('load_entry_point', importlib_load_entry_point) + def main(): sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) _args = [sys.argv[0], 'demo'] + sys.argv[1:] sys.argv = _args sys.exit(load_entry_point('netfoundry', 'console_scripts', 'nfctl')()) + if __name__ == '__main__': main() From 0d3916ea2dec51d23d22142579a0d20ed5dd8d52 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Thu, 13 Nov 2025 13:43:57 -0500 Subject: [PATCH 34/41] require requests versions after the breaking change --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 14059a7..b01084a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,7 +30,7 @@ install_requires = pygments >= 2.11 pyjwt >= 2.3 pyyaml >= 5.4 - requests >= 2.27 + requests >= 2.30 tabulate >= 0.8 requests-cache >= 0.9.4 setup_requires = From 5c2c12f1495d59afd9a68ee02fc97be357a7e20d Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Thu, 13 Nov 2025 14:37:44 -0500 Subject: [PATCH 35/41] run ci test in aws --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 143c148..6db9006 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -87,8 +87,8 @@ jobs: general.verbose=yes || true # FIXME: sometimes config command exits with an error nfctl demo \ --size medium \ - --regions us-ashburn-1 us-phoenix-1 \ - --provider OCI + --regions us-west-2 us-east-1 \ + --provider AWS nfctl \ list services nfctl \ From 6bb36766f6b0003cd710406b3bff2944d31b36d6 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Thu, 13 Nov 2025 16:34:16 -0500 Subject: [PATCH 36/41] contrain milc version --- .github/workflows/main.yml | 2 +- netfoundry/ctl.py | 9 +++++---- netfoundry/organization.py | 41 +++++++++++++++++++++++--------------- netfoundry/utility.py | 23 ++++++++++++--------- setup.cfg | 2 +- 5 files changed, 46 insertions(+), 31 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6db9006..ff83c52 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -85,7 +85,7 @@ jobs: general.network=$(nfctl demo --echo-name --prefix 'gh-${{ github.run_id }}') \ general.yes=True \ general.verbose=yes || true # FIXME: sometimes config command exits with an error - nfctl demo \ + nfctl --wait 3000 demo \ --size medium \ --regions us-west-2 us-east-1 \ --provider AWS diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index 38b19be..b1396d6 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -44,7 +44,7 @@ # import milc cli from milc import cli, questions # noqa: E402 -# set milc options using new API +# set milc options (requires milc >= 1.8.0) cli.milc_options(name='nfctl', author='NetFoundry', version=f'v{netfoundry_version}') # this creates the config subcommand from milc.subcommand import config # noqa: F401,E402 @@ -994,7 +994,7 @@ def demo(cli): spinner.text = f"Waiting for {len(hosted_edge_routers)} hosted router(s) to provision" with spinner: for router in hosted_edge_routers: - network.wait_for_statuses(expected_statuses=RESOURCES["edge-routers"].status_symbols["complete"], id=router['id'], type="edge-router", wait=2222, progress=False) + network.wait_for_statuses(expected_statuses=RESOURCES["edge-routers"].status_symbols["complete"], id=router['id'], type="edge-router", wait=cli.config.general.wait, progress=False) # ensure the router tunneler is available # network.wait_for_entity_name_exists(entity_name=router['name'], entity_type='endpoint') # router_tunneler = network.find_resources(type='endpoint', name=router['name'])[0] @@ -1105,8 +1105,9 @@ def demo(cli): customer_router = network.edge_routers(name=customer_router_name)[0] spinner.succeed(sub("Finding", "Found", spinner.text)) - spinner.text = f"Waiting for customer router {customer_router_name} to be ready for registration" - # wait for customer router to be PROVISIONED so that registration will be available + spinner.text = f"Getting registration key for customer router {customer_router_name}" + # Customer routers don't auto-provision - registration key is available immediately at status NEW + # The router will only reach PROVISIONED status after manual registration and connection with spinner: try: network.wait_for_statuses(expected_statuses=RESOURCES["edge-routers"].status_symbols["complete"], id=customer_router['id'], type="edge-router", wait=222, progress=False) diff --git a/netfoundry/organization.py b/netfoundry/organization.py index 492342f..91cbe3a 100644 --- a/netfoundry/organization.py +++ b/netfoundry/organization.py @@ -159,6 +159,13 @@ def __init__(self, self.expiry_seconds = round(self.expiry - epoch) self.audience = token_cache['audience'] + # Check if cached token is expired + if self.expiry_seconds < 0: + self.logger.debug(f"cached token is expired ({self.expiry_seconds}s ago), forcing renewal") + self.token = None + self.expiry = None + self.audience = None + # if the token was found but not the expiry then try to parse to extract the expiry so we can enforce minimum lifespan seconds if self.token and not self.expiry: try: @@ -280,14 +287,16 @@ def __init__(self, self.expiry_seconds = round(self.expiry - epoch) self.logger.debug(f"bearer token expiry in {self.expiry_seconds}s") - # renew token if not existing or imminent expiry, else continue - if not self.token or self.expiry_seconds < expiry_minimum: + # renew token if not existing, expired, or imminent expiry, else continue + if not self.token or self.expiry_seconds < 0 or self.expiry_seconds < expiry_minimum: # we've already done the work to determine the cached token is expired or imminently-expiring, might as well save other runs the same trouble self.logout() self.expiry = None self.audience = None if self.token and self.expiry_seconds < expiry_minimum: self.logger.debug(f"token expiry {self.expiry_seconds}s is less than configured minimum {expiry_minimum}s") + if self.expiry_seconds < 0: + self.logger.debug(f"token is expired ({abs(self.expiry_seconds)}s ago), forcing renewal") if not credentials_configured: raise NFAPINoCredentials("unable to renew because credentials are not configured") else: @@ -430,7 +439,7 @@ def get_caller_identity(self): except Exception as e: self.logger.debug(f"failed to get caller identity from url: '{url}', trying next until last, caught {e}") else: - return(caller) + return caller raise RuntimeError("failed to get caller identity from any url") def get_identity(self, identity_id: str): @@ -444,7 +453,7 @@ def get_identity(self, identity_id: str): except Exception as e: raise RuntimeError(f"failed to get identity from url: '{url}', caught {e}") else: - return(identity) + return identity def find_identities(self, type: str = 'identities', **kwargs): """Get identities as a collection. @@ -473,7 +482,7 @@ def find_identities(self, type: str = 'identities', **kwargs): except Exception as e: raise RuntimeError(f"failed to get identities from url: '{url}', caught {e}") else: - return(identities) + return identities get_identities = find_identities def find_roles(self, **kwargs): @@ -503,7 +512,7 @@ def find_roles(self, **kwargs): except Exception as e: raise RuntimeError(f"failed to get roles from url: '{url}', caught {e}") else: - return(roles) + return roles def get_role(self, role_id: str): """Get roles as a collection.""" @@ -514,7 +523,7 @@ def get_role(self, role_id: str): except Exception as e: raise RuntimeError(f"failed to get role from url: '{url}', caught {e}") else: - return(role) + return role def find_organizations(self, **kwargs): """Find organizations as a collection. @@ -536,7 +545,7 @@ def find_organizations(self, **kwargs): except Exception as e: raise RuntimeError(f"failed to get organizations from url: '{url}', caught {e}") else: - return(organizations) + return organizations get_organizations = find_organizations def get_organization(self, id): @@ -551,7 +560,7 @@ def get_organization(self, id): except Exception as e: raise RuntimeError(f"failed to get organization from url: '{url}', caught {e}") else: - return(organization) + return organization def get_network_group(self, network_group_id): """ @@ -565,7 +574,7 @@ def get_network_group(self, network_group_id): except Exception as e: raise RuntimeError(f"failed to get network_group from url: '{url}', caught {e}") else: - return(network_group) + return network_group def get_network(self, network_id: str, embed: object = None, accept: str = None): """Describe a Network by ID. @@ -593,7 +602,7 @@ def get_network(self, network_id: str, embed: object = None, accept: str = None) url = self.audience+'core/v2/networks/'+network_id network, status_symbol = get_generic_resource_by_url(setup=self, url=url, accept=accept, **params) - return(network) + return network def find_network_groups_by_organization(self, **kwargs): """Find network groups as a collection. @@ -604,7 +613,7 @@ def find_network_groups_by_organization(self, **kwargs): network_groups = list() for i in find_generic_resources(setup=self, url=url, embedded=RESOURCES['network-groups']._embedded, **kwargs): network_groups.extend(i) - return(network_groups) + return network_groups get_network_groups_by_organization = find_network_groups_by_organization network_groups = get_network_groups_by_organization @@ -631,7 +640,7 @@ def find_networks_by_organization(self, name: str = None, deleted: bool = False, except Exception as e: raise RuntimeError(f"failed to get networks from url: '{url}', caught {e}") else: - return(networks) + return networks get_networks_by_organization = find_networks_by_organization def network_exists(self, name: str, deleted: bool = False): @@ -641,9 +650,9 @@ def network_exists(self, name: str, deleted: bool = False): :param deleted: include deleted networks in results """ if self.count_networks_with_name(name=name, deleted=deleted) > 0: - return(True) + return True else: - return(False) + return False def count_networks_with_name(self, name: str, deleted: bool = False, unique: bool = True): """ @@ -686,5 +695,5 @@ def find_networks_by_group(self, network_group_id: str, deleted: bool = False, a except Exception as e: raise RuntimeError(f"failed to get networks from url: '{url}', caught {e}") else: - return(networks) + return networks get_networks_by_group = find_networks_by_group diff --git a/netfoundry/utility.py b/netfoundry/utility.py index dbcfa87..7d4bca4 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -43,15 +43,15 @@ def plural(singular): # if already plural then return, else pluralize p = inflect.engine() if singular[-1:] == 's': - return(singular) + return singular else: - return(p.plural_noun(singular)) + return p.plural_noun(singular) def singular(plural): """Singularize a plural form.""" p = inflect.engine() - return(p.singular_noun(plural)) + return p.singular_noun(plural) def kebab2camel(kebab: str, case: str = "lower"): # "lower" dromedary or "upper" Pascal @@ -485,12 +485,12 @@ def find_generic_resources(setup: object, url: str, headers: dict = dict(), embe ) response.raise_for_status() resource_page = response.json() - + # Handle non-paginated endpoints that return direct lists if isinstance(resource_page, list): yield resource_page return - + if isinstance(resource_page, dict) and resource_page.get('page'): try: total_pages = resource_page['page']['totalPages'] @@ -623,9 +623,7 @@ class ResourceType(ResourceTypeParent): embeddable: bool # legal to request embedding in a parent resource in same domain parent: str = field(default=str()) # optional parent ResourceType instance name status: str = field(default='status') # name of property where symbolic status is expressed - _embedded: str = field(default='default') # the key under which lists are found in the API - # e.g. networkControllerList (computed if not provided as dromedary - # case singular) + _embedded: str = field(default='default') # the key under which lists are found in the API e.g. networkControllerList (computed if not provided as dromedary case singular) create_responses: list = field(default_factory=list) # expected HTTP response codes for create operation no_update_props: list = field(default_factory=list) # expected HTTP response codes for create operation create_template: dict = field(default_factory=lambda: { @@ -932,7 +930,14 @@ def send(self, request, **kwargs): http = Session() # no cache HTTP_CACHE_EXPIRE = 33 -http_cache = CachedSession(cache_name=f"{get_user_cache_dir()}/http_cache", backend='sqlite', expire_after=HTTP_CACHE_EXPIRE) +http_cache = CachedSession( + cache_name=f"{get_user_cache_dir()}/http_cache_tz", + backend='sqlite', + expire_after=HTTP_CACHE_EXPIRE, + allowable_codes=(200, 203, 300, 301, 308), + timeout=DEFAULT_TIMEOUT, + stale_if_error=True +) # Mount it for both http and https usage adapter = TimeoutHTTPAdapter(timeout=DEFAULT_TIMEOUT, max_retries=RETRY_STRATEGY) http.mount("https://", adapter) diff --git a/setup.cfg b/setup.cfg index b01084a..c5105cd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,7 @@ include_package_data = True packages = find: install_requires = inflect >= 5.3 - milc >= 1.6.6 + milc >= 1.8.0 packaging >= 20.9 platformdirs >= 2.4 pygments >= 2.11 From 61f15fea66e714a7ac0e90717ae921b02c84b345 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Thu, 13 Nov 2025 16:52:36 -0500 Subject: [PATCH 37/41] parallelize demo routers --- netfoundry/ctl.py | 75 +++++++++++++++++++++++++++++++---------------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index b1396d6..d8476c8 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -13,6 +13,7 @@ import signal import jwt import tempfile +from concurrent.futures import ThreadPoolExecutor, as_completed from json import dumps as json_dumps from json import load as json_load from json import loads as json_loads @@ -22,6 +23,7 @@ from subprocess import CalledProcessError from sys import exit as sysexit from sys import stderr, stdin, stdout +from threading import Lock from xml.sax.xmlreader import InputSource from jwt.exceptions import PyJWTError @@ -961,32 +963,55 @@ def demo(cli): else: spinner.succeed(f"Found a hosted router in {region}") - spinner.text = f"Creating {len(fabric_placements)} hosted router(s)" - with spinner: - for region in fabric_placements: - er_name = f"Hosted Router {region} [{cli.config.demo.provider}]" - if not network.edge_router_exists(er_name): - er = network.create_edge_router( - name=er_name, - attributes=[ - "#hosted_routers", - "#demo_exits", - f"#{cli.config.demo.provider}", - ], - provider=cli.config.demo.provider, - location_code=region, - tunneler_enabled=False, # workaround for MOP-18098 (missing tunneler binding in ziti-router config) - ) - hosted_edge_routers.extend([er]) - spinner.succeed(f"Created {cli.config.demo.provider} router in {region}") + # Helper function to create or validate a single router (runs in parallel) + def create_or_validate_router(region): + """Create or validate router for a region. Returns (region, router_dict, message).""" + er_name = f"Hosted Router {region} [{cli.config.demo.provider}]" + if not network.edge_router_exists(er_name): + er = network.create_edge_router( + name=er_name, + attributes=[ + "#hosted_routers", + "#demo_exits", + f"#{cli.config.demo.provider}", + ], + provider=cli.config.demo.provider, + location_code=region, + tunneler_enabled=False, # workaround for MOP-18098 (missing tunneler binding in ziti-router config) + ) + message = f"Created {cli.config.demo.provider} router in {region}" + return (region, er, message) + else: + er_matches = network.edge_routers(name=er_name, only_hosted=True) + if len(er_matches) == 1: + er = er_matches[0] else: - er_matches = network.edge_routers(name=er_name, only_hosted=True) - if len(er_matches) == 1: - er = er_matches[0] - else: - raise RuntimeError(f"unexpectedly found more than one matching router for name '{er_name}'") - if er['status'] in RESOURCES["edge-routers"].status_symbols["error"] + RESOURCES["edge-routers"].status_symbols["deleting"] + RESOURCES["edge-routers"].status_symbols["deleted"]: - raise RuntimeError(f"hosted router '{er_name}' has unexpected status '{er['status']}'") + raise RuntimeError(f"unexpectedly found more than one matching router for name '{er_name}'") + if er['status'] in RESOURCES["edge-routers"].status_symbols["error"] + RESOURCES["edge-routers"].status_symbols["deleting"] + RESOURCES["edge-routers"].status_symbols["deleted"]: + raise RuntimeError(f"hosted router '{er_name}' has unexpected status '{er['status']}'") + return (region, er, None) # No message for existing routers + + # Parallelize router creation with thread-safe spinner updates + spinner.text = f"Creating {len(fabric_placements)} hosted router(s)" + spinner_lock = Lock() + new_routers = [] + + with ThreadPoolExecutor(max_workers=min(len(fabric_placements), 5)) as executor: + # Submit all router creation tasks + future_to_region = {executor.submit(create_or_validate_router, region): region for region in fabric_placements} + + # Collect results as they complete + for future in as_completed(future_to_region): + region, er, message = future.result() + new_routers.append(er) + + # Thread-safe spinner update for newly created routers + if message: + with spinner_lock: + spinner.succeed(message) + + # Add all new routers to the list + hosted_edge_routers.extend(new_routers) if not len(hosted_edge_routers) > 0: raise RuntimeError("unexpected problem with router placements, found zero hosted routers") From fb98e886387f19054af3a95e82cc3cf5e84c92fe Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Thu, 13 Nov 2025 16:52:46 -0500 Subject: [PATCH 38/41] cast wait type --- netfoundry/ctl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index d8476c8..16ca004 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -96,7 +96,7 @@ def __call__(self, parser, namespace, values, option_string=None): @cli.argument('-B', '--borders', default=True, action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fstore_boolean", help='print cell borders in text tables') @cli.argument('-H', '--headers', default=True, action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fstore_boolean", help='print column headers in text tables') @cli.argument('-Y', '--yes', action="iframe.php?url=https%3A%2F%2Fgithub.com%2Fstore_true", arg_only=True, help='answer yes to potentially-destructive operations') -@cli.argument('-W', '--wait', help='seconds to wait for long-running processes to finish', default=900) +@cli.argument('-W', '--wait', type=int, help='seconds to wait for long-running processes to finish', default=900) @cli.argument('--proxy', help=argparse.SUPPRESS) @cli.argument('--gateway', default="gateway", help=argparse.SUPPRESS) @cli.entrypoint('configure the CLI to manage a network') From 0df14072ce4da406d6b1adb6c9127e380ea016db Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Fri, 14 Nov 2025 13:15:00 -0500 Subject: [PATCH 39/41] consolidate local/ci test steps in a shell script --- .github/workflows/main.yml | 23 +---------- scripts/README.md | 85 ++++++++++++++++++++++++++++++++++++++ scripts/test-demo.sh | 66 +++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 22 deletions(-) create mode 100644 scripts/README.md create mode 100755 scripts/test-demo.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ff83c52..c8b7b8e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -77,28 +77,7 @@ jobs: NETFOUNDRY_CLIENT_ID: ${{ secrets.NETFOUNDRY_CLIENT_ID }} NETFOUNDRY_PASSWORD: ${{ secrets.NETFOUNDRY_PASSWORD }} NETFOUNDRY_OAUTH_URL: ${{ secrets.NETFOUNDRY_OAUTH_URL }} - run: | - set -o xtrace - set -o pipefail - - nfctl config \ - general.network=$(nfctl demo --echo-name --prefix 'gh-${{ github.run_id }}') \ - general.yes=True \ - general.verbose=yes || true # FIXME: sometimes config command exits with an error - nfctl --wait 3000 demo \ - --size medium \ - --regions us-west-2 us-east-1 \ - --provider AWS - nfctl \ - list services - nfctl \ - get service name=echo% > /tmp/echo.yml - nfctl \ - delete service name=echo% - nfctl \ - create service --file /tmp/echo.yml - nfctl \ - delete network + run: ./scripts/test-demo.sh - name: Publish Test Package uses: pypa/gh-action-pypi-publish@v1.13.0 diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 0000000..2902190 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,85 @@ +# Test Scripts + +## test-demo.sh + +Test script for the `nfctl demo` command. Can be run locally or in GitHub Actions. + +### Usage + +**In GitHub Actions:** + +```yaml +- name: Run demo test + env: + NETFOUNDRY_CLIENT_ID: ${{ secrets.NETFOUNDRY_CLIENT_ID }} + NETFOUNDRY_PASSWORD: ${{ secrets.NETFOUNDRY_PASSWORD }} + NETFOUNDRY_OAUTH_URL: ${{ secrets.NETFOUNDRY_OAUTH_URL }} + run: ./scripts/test-demo.sh +``` + +The script automatically detects GitHub Actions via `GITHUB_RUN_ID` and uses it in the network name prefix. + +**Locally:** + +```bash +# Use default prefix (local-) +./scripts/test-demo.sh + +# Use custom prefix +DEMO_PREFIX=mytest ./scripts/test-demo.sh + +# Specify organization and network group +NETFOUNDRY_ORGANIZATION=acme \ +NETFOUNDRY_NETWORK_GROUP=testing \ +DEMO_PREFIX=mytest \ +./scripts/test-demo.sh +``` + +### What it does + +1. Creates a temporary directory and config file (cleaned up on exit) +2. Generates a unique network name using `--echo-name` +3. Configures nfctl with all settings in the temp config: + - Network name (generated) + - Organization (from `NETFOUNDRY_ORGANIZATION` if set) + - Network group (from `NETFOUNDRY_NETWORK_GROUP` if set) + - Auto-confirm and verbose flags +4. Runs the demo with medium size, AWS provider, us-west-2 and us-east-1 regions +5. Tests service operations (list, get, delete, create) +6. Cleans up by deleting the network and removing temp directory + +### Environment Variables + +**Script Configuration:** + +- `GITHUB_RUN_ID` - Auto-detected in GitHub Actions, used for network prefix +- `DEMO_PREFIX` - Custom prefix for local runs (default: `local-`) +- `NETFOUNDRY_PROFILE` - Profile name for token cache isolation (default: `default`) + +**Standard NetFoundry Environment Variables:** + +- `NETFOUNDRY_ORGANIZATION` - Optional organization name (omitted if unset) +- `NETFOUNDRY_NETWORK_GROUP` - Optional network group name (omitted if unset) +- `NETFOUNDRY_CLIENT_ID` - NetFoundry API credentials +- `NETFOUNDRY_PASSWORD` - NetFoundry API credentials +- `NETFOUNDRY_OAUTH_URL` - NetFoundry OAuth URL +- `NETFOUNDRY_API_ACCOUNT` - Path to API credentials JSON file + +These standard variables match those used by `nfctl login --eval` for consistency. + +**Profile Usage:** + +The `NETFOUNDRY_PROFILE` variable allows you to isolate token caches for different accounts. Each profile uses a separate cache file (`~/.cache/netfoundry/.json`), preventing conflicts when working with multiple NetFoundry accounts. + +```bash +# Use a specific profile +NETFOUNDRY_PROFILE=advdev \ +NETFOUNDRY_API_ACCOUNT=~/.config/netfoundry/advdev.json \ +./scripts/test-demo.sh +``` + +### Features + +- **Isolated config**: Each run uses a temporary config file that doesn't interfere with your existing nfctl configuration +- **Auto-cleanup**: Temporary directory is automatically removed on exit (success or failure) +- **Config-based scoping**: Organization and network group are set in the config file (from environment variables) rather than passed as CLI options on every command diff --git a/scripts/test-demo.sh b/scripts/test-demo.sh new file mode 100755 index 0000000..dd2b576 --- /dev/null +++ b/scripts/test-demo.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Test script for nfctl demo command +# Can be run locally or in GitHub Actions + +set -o xtrace +set -o pipefail + +# Create temporary directory and config file +TEMP_DIR=$(mktemp -d) +TEMP_CONFIG="${TEMP_DIR}/nfctl.ini" +echo "Using temporary config: ${TEMP_CONFIG}" + +# Cleanup on exit +trap 'rm -rf "${TEMP_DIR}"' EXIT + +# Determine prefix based on environment +if [[ -n "${GITHUB_RUN_ID}" ]]; then + # Running in GitHub Actions + PREFIX="gh-${GITHUB_RUN_ID}" +else + # Running locally - use timestamp or custom prefix + PREFIX="${DEMO_PREFIX:-local-$(date +%s)}" +fi + +echo "Using demo prefix: ${PREFIX} (override with DEMO_PREFIX)" + +# Set profile (default: "default") +: "${NETFOUNDRY_PROFILE:=default}" +echo "Using profile: ${NETFOUNDRY_PROFILE} (override with NETFOUNDRY_PROFILE)" + +# Helper function to run nfctl with the temp config and profile +nfctl() { + command nfctl --profile "${NETFOUNDRY_PROFILE}" --config-file "${TEMP_CONFIG}" "$@" +} + +# Configure nfctl with generated network name and basic settings +nfctl config \ + "general.network=$(command nfctl demo --echo-name --prefix "${PREFIX}")" \ + general.yes=True \ + general.verbose=yes || true # FIXME: sometimes config command exits with an error + +# Set optional organization and network group from standard NetFoundry env vars +if [[ -n "${NETFOUNDRY_ORGANIZATION}" ]]; then + nfctl config "general.organization=${NETFOUNDRY_ORGANIZATION}" +fi +if [[ -n "${NETFOUNDRY_NETWORK_GROUP}" ]]; then + nfctl config "general.network_group=${NETFOUNDRY_NETWORK_GROUP}" +fi + +# Run the demo +nfctl --wait 3000 demo \ + --size medium \ + --regions us-west-2 us-east-1 \ + --provider AWS + +# Test service operations +nfctl list services + +nfctl get service name=echo% > /tmp/echo.yml + +nfctl delete service name=echo% + +nfctl create service --file /tmp/echo.yml + +# Cleanup: delete the network +nfctl delete network From 75644d4579ba62db0f486f39a8e782aa78b756cb Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Fri, 14 Nov 2025 13:15:15 -0500 Subject: [PATCH 40/41] print http response body if error --- netfoundry/network.py | 8 ++++++++ netfoundry/utility.py | 27 ++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/netfoundry/network.py b/netfoundry/network.py index 0018e4e..5651210 100644 --- a/netfoundry/network.py +++ b/netfoundry/network.py @@ -458,6 +458,14 @@ def patch_resource(self, patch: dict, type: str = None, id: str = None, wait: in headers=headers, json=pruned_patch ) + if after_response.status_code in range(400, 600): + self.logger.debug( + '%s\n%s %s\r\n%s\r\n\r\n%s', + '-----------RESPONSE-----------', + after_response.status_code, after_response.reason, + '\r\n'.join('{}: {}'.format(k, v) for k, v in after_response.headers.items()), + after_response.text + ) after_response.raise_for_status() # raise any gross errors immediately after_response_code = after_response.status_code if after_response_code in [STATUS_CODES.codes.OK, STATUS_CODES.codes.ACCEPTED]: diff --git a/netfoundry/utility.py b/netfoundry/utility.py index 7d4bca4..a059a54 100644 --- a/netfoundry/utility.py +++ b/netfoundry/utility.py @@ -322,12 +322,17 @@ def create_generic_resource(setup: object, url: str, body: dict, headers: dict = if response.status_code in range(400, 600): req = response.request setup.logger.debug( - '%s\n%s\r\n%s\r\n\r\n%s', - '-----------START-----------', + '%s\n%s\r\n%s\r\n\r\n%s\n%s\n%s %s\r\n%s\r\n\r\n%s', + '-----------REQUEST-----------', req.method + ' ' + req.url, '\r\n'.join('{}: {}'.format(k, v) for k, v in req.headers.items()), - req.body + req.body, + '-----------RESPONSE-----------', + response.status_code, response.reason, + '\r\n'.join('{}: {}'.format(k, v) for k, v in response.headers.items()), + response.text ) + setup.logger.error(f"HTTP {response.status_code} error response body: {response.text}") response.raise_for_status() resource = response.json() @@ -388,6 +393,14 @@ def get_generic_resource_by_url(setup: object, url: str, headers: dict = dict(), try: response.raise_for_status() except HTTPError: + if response.status_code in range(400, 600): + setup.logger.debug( + '%s\n%s %s\r\n%s\r\n\r\n%s', + '-----------RESPONSE-----------', + response.status_code, response.reason, + '\r\n'.join('{}: {}'.format(k, v) for k, v in response.headers.items()), + response.text + ) if resource_type.name in ["process-executions"] and status_symbol == "FORBIDDEN": # FIXME: MOP-18095 workaround the create network process ID mismatch bug url_parts = urlparse(url) path_parts = url_parts.path.split('/') @@ -483,6 +496,14 @@ def find_generic_resources(setup: object, url: str, headers: dict = dict(), embe proxies=setup.proxies, verify=setup.verify, ) + if response.status_code in range(400, 600): + setup.logger.debug( + '%s\n%s %s\r\n%s\r\n\r\n%s', + '-----------RESPONSE-----------', + response.status_code, response.reason, + '\r\n'.join('{}: {}'.format(k, v) for k, v in response.headers.items()), + response.text + ) response.raise_for_status() resource_page = response.json() From ed661102f2603032748aebc327bcadf3a7db64e3 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Fri, 14 Nov 2025 13:43:59 -0500 Subject: [PATCH 41/41] delete the private 'branch' router part of the demo because it no longer works reliably with mop --- .github/workflows/main.yml | 41 ++++++++++++++++++++++++++++++++++++++ netfoundry/ctl.py | 26 ------------------------ scripts/test-demo.sh | 1 + 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c8b7b8e..d48cf88 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -72,6 +72,7 @@ jobs: register-python-argcomplete nfctl - name: Run the NF CLI demo to test installed version + id: test_demo shell: bash env: NETFOUNDRY_CLIENT_ID: ${{ secrets.NETFOUNDRY_CLIENT_ID }} @@ -141,3 +142,43 @@ jobs: platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.compose_tags.outputs.container_tags }} + + cleanup-delay: + if: failure() + needs: [build_pypi_and_docker] + runs-on: ubuntu-latest + steps: + - name: Wait 30 minutes before cleanup + run: | + echo "Test demo failed to complete. Waiting 30 minutes before cleanup to allow investigation..." + sleep 1800 + + cleanup-network: + if: always() && needs.build_pypi_and_docker.result == 'failure' + needs: [cleanup-delay] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.12' + + - name: Install nfctl + run: | + python -m pip install --upgrade pip + pip install . + + - name: Delete test network + env: + NETFOUNDRY_CLIENT_ID: ${{ secrets.NETFOUNDRY_CLIENT_ID }} + NETFOUNDRY_PASSWORD: ${{ secrets.NETFOUNDRY_PASSWORD }} + NETFOUNDRY_OAUTH_URL: ${{ secrets.NETFOUNDRY_OAUTH_URL }} + run: | + # Use wildcard pattern to match network created by this run + NETWORK_PATTERN="gh-${GITHUB_RUN_ID}-%" + echo "Attempting to delete network matching: ${NETWORK_PATTERN}" + + # Try to delete the network, ignore errors if it doesn't exist + nfctl delete network "name=${NETWORK_PATTERN}" --yes || echo "Network may not exist or already deleted" diff --git a/netfoundry/ctl.py b/netfoundry/ctl.py index 16ca004..f596333 100644 --- a/netfoundry/ctl.py +++ b/netfoundry/ctl.py @@ -1116,32 +1116,6 @@ def create_or_validate_router(region): services[svc]['properties'] = network.services(name=svc)[0] spinner.succeed(sub("Finding", "Found", spinner.text)) - # create a customer-hosted ER unless exists - customer_router_name = "Branch Exit Router" - spinner.text = f"Finding customer router '{customer_router_name}'" - with spinner: - if not network.edge_router_exists(name=customer_router_name): - spinner.text = sub("Finding", "Creating", spinner.text) - customer_router = network.create_edge_router( - name=customer_router_name, - attributes=["#branch_exit_routers"], - tunneler_enabled=True) - else: - customer_router = network.edge_routers(name=customer_router_name)[0] - spinner.succeed(sub("Finding", "Found", spinner.text)) - - spinner.text = f"Getting registration key for customer router {customer_router_name}" - # Customer routers don't auto-provision - registration key is available immediately at status NEW - # The router will only reach PROVISIONED status after manual registration and connection - with spinner: - try: - network.wait_for_statuses(expected_statuses=RESOURCES["edge-routers"].status_symbols["complete"], id=customer_router['id'], type="edge-router", wait=222, progress=False) - customer_router_registration = network.rotate_edge_router_registration(id=customer_router['id']) - except Exception as e: - raise RuntimeError(f"error getting router registration, got {e}") - else: - spinner.succeed(f"Customer router ready to register with key '{customer_router_registration['registrationKey']}'") - # create unless exists app_wan_name = "Default Service Policy" spinner.text = "Finding service policy" diff --git a/scripts/test-demo.sh b/scripts/test-demo.sh index dd2b576..a093dd6 100755 --- a/scripts/test-demo.sh +++ b/scripts/test-demo.sh @@ -2,6 +2,7 @@ # Test script for nfctl demo command # Can be run locally or in GitHub Actions +set -o errexit set -o xtrace set -o pipefail