Source code for google.cloud.forseti.services.model.importer.importer

# Copyright 2017 The Forseti Security Authors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Importer implementations."""

# pylint: disable=too-many-lines
# pylint: disable=too-many-instance-attributes

from builtins import object
import json
from io import StringIO
import traceback

from future import standard_library
from sqlalchemy.exc import SQLAlchemyError

from google.cloud.forseti.common.util import logger
from google.cloud.forseti.services.inventory.storage import Categories
from google.cloud.forseti.services.inventory.storage import DataAccess
from google.cloud.forseti.services.utils import get_resource_id_from_type_name
from google.cloud.forseti.services.utils import get_sql_dialect
from google.cloud.forseti.services.utils import to_full_resource_name
from google.cloud.forseti.services.utils import to_type_name

standard_library.install_aliases()

LOGGER = logger.get_logger(__name__)

GCP_TYPE_LIST = [
    'appengine_app',
    'appengine_instance',
    'appengine_service',
    'appengine_version',
    'backendservice',
    'bigquery_table',
    'billing_account',
    'bucket',
    'cloudsqlinstance',
    'composite_root',
    'compute_autoscaler',
    'compute_backendbucket',
    'compute_healthcheck',
    'compute_httphealthcheck',
    'compute_httpshealthcheck',
    'compute_license',
    'compute_project',
    'compute_router',
    'compute_sslcertificate',
    'compute_targethttpproxy',
    'compute_targethttpsproxy',
    'compute_targetinstance',
    'compute_targetpool',
    'compute_targetsslproxy',
    'compute_targettcpproxy',
    'compute_targetvpngateway',
    'compute_urlmap',
    'compute_vpntunnel',
    'crm_org_policy',
    'dataproc_cluster',
    'dataset',
    'disk',
    'dns_managedzone',
    'dns_policy',
    'firewall',
    'folder',
    'forwardingrule',
    'image',
    'instance',
    'instancegroup',
    'instancegroupmanager',
    'instancetemplate',
    'kms_cryptokey',
    'kms_cryptokeyversion',
    'kms_keyring',
    'kubernetes_cluster',
    'kubernetes_clusterrole',
    'kubernetes_clusterrolebinding',
    'kubernetes_namespace',
    'kubernetes_node',
    'kubernetes_pod',
    'kubernetes_role',
    'kubernetes_rolebinding',
    'lien',
    'network',
    'organization',
    'project',
    'pubsub_subscription',
    'pubsub_topic',
    'serviceaccount',
    'serviceaccount_key',
    'sink',
    'snapshot',
    'spanner_database',
    'spanner_instance',
    'subnetwork',
]

GSUITE_TYPE_LIST = [
    'gsuite_group',
    'gsuite_user',
]

MEMBER_TYPE_LIST = [
    'gsuite_user_member',
    'gsuite_group_member',
]

GROUPS_SETTINGS_LIST = [
    'gsuite_groups_settings',
]


[docs]class ResourceCache(dict): """Resource cache."""
[docs] def __setitem__(self, key, value): """Overriding to assert the keys does not exist previously. Args: key (object): Key into the dict. value (object): Value to set. Raises: Exception: If the key already exists in the dict. """ if key in self: raise Exception('Key should not exist: {}'.format(key)) super(ResourceCache, self).__setitem__(key, value)
[docs]class EmptyImporter(object): """Imports an empty model.""" def __init__(self, session, readonly_session, model, dao, _, *args, **kwargs): """Create an EmptyImporter which creates an empty stub model. Args: session (object): Database session. readonly_session (Session): Database session (read-only). model (str): Model name to create. dao (object): Data Access Object from dao.py. _ (object): Unused. *args (list): Unused. **kwargs (dict): Unused. """ del args, kwargs # Unused. self.session = session self.readonly_session = readonly_session self.model = model self.dao = dao
[docs] def run(self): """Runs the import.""" self.session.add(self.model) self.model.add_description( json.dumps( {'source': 'empty', 'pristine': True}, sort_keys=True ) ) self.model.set_done() self.session.commit()
[docs]class InventoryImporter(object): """Imports data from Inventory.""" def __init__(self, session, readonly_session, model, dao, service_config, inventory_index_id, *args, **kwargs): """Create a Inventory importer which creates a model from the inventory. Args: session (Session): Database session. readonly_session (Session): Database session (read-only). model (Model): Model object. dao (object): Data Access Object from dao.py service_config (ServiceConfig): Service configuration. inventory_index_id (int64): Inventory id to import from *args (list): Unused. **kwargs (dict): Unused. """ del args, kwargs # Unused. self.readonly_session = readonly_session self.session = session self.model = model self.dao = dao self.service_config = service_config self.inventory_index_id = inventory_index_id self.session.add(self.model) self.role_cache = {} self.permission_cache = {} self.resource_cache = ResourceCache() self.membership_items = [] self.membership_map = {} # Maps group_name to {member_name} self.member_cache = {} self.member_cache_policies = {} self.groups_settings_cache = set() self.found_root = False
[docs] def _flush_session(self): """Flush the session with rollback on errors.""" try: self.session.flush() except SQLAlchemyError: LOGGER.exception( 'Unexpected SQLAlchemyError occurred during model creation.') self.session.rollback()
[docs] def _commit_session(self): """Commit the session with rollback on errors.""" try: self.session.commit() except SQLAlchemyError: LOGGER.exception( 'Unexpected SQLAlchemyError occurred during model creation.') self.session.rollback()
# pylint: disable=too-many-statements
[docs] def run(self): """Runs the import. Raises: NotImplementedError: If the importer encounters an unknown inventory type. """ autocommit = self.session.autocommit autoflush = self.session.autoflush try: self.session.autocommit = False self.session.autoflush = True root = DataAccess.get_root(self.readonly_session, self.inventory_index_id) inventory_index = DataAccess.get(self.readonly_session, self.inventory_index_id) description = { 'source': 'inventory', 'source_info': { 'inventory_index_id': self.inventory_index_id}, 'source_root': self._type_name(root), 'pristine': True, 'gsuite_enabled': DataAccess.type_exists( self.readonly_session, self.inventory_index_id, ['gsuite_group', 'gsuite_user']) } LOGGER.debug('Model description: %s', description) self.model.add_description(json.dumps(description, sort_keys=True)) if root.get_resource_type() in ['organization']: LOGGER.debug('Root resource is organization: %s', root) else: LOGGER.debug('Root resource is not organization: %s.', root) item_counter = 0 LOGGER.debug('Start storing resources into models.') for resource in DataAccess.iter(self.readonly_session, self.inventory_index_id, GCP_TYPE_LIST): item_counter += 1 self._store_resource(resource) if not item_counter % 1000: # Flush database every 1000 resources LOGGER.debug('Flushing model write session: %s', item_counter) self._flush_session() if not item_counter % 100000: # Commit every 100k resources while iterating # through all the resources. LOGGER.debug('Commiting model write session: %s', item_counter) self._commit_session() self._commit_session() LOGGER.debug('Finished storing resources into models.') item_counter += self.model_action_wrapper( DataAccess.iter(self.readonly_session, self.inventory_index_id, ['role']), self._convert_role ) item_counter += self.model_action_wrapper( DataAccess.iter(self.readonly_session, self.inventory_index_id, GCP_TYPE_LIST, fetch_category=Categories.dataset_policy), self._convert_dataset_policy ) item_counter += self.model_action_wrapper( DataAccess.iter(self.readonly_session, self.inventory_index_id, GCP_TYPE_LIST, fetch_category=Categories.gcs_policy), self._convert_gcs_policy ) item_counter += self.model_action_wrapper( DataAccess.iter( self.readonly_session, self.inventory_index_id, GCP_TYPE_LIST, fetch_category=Categories.kubernetes_service_config), self._convert_service_config ) self.model_action_wrapper( DataAccess.iter(self.readonly_session, self.inventory_index_id, GSUITE_TYPE_LIST), self._store_gsuite_principal ) self.model_action_wrapper( DataAccess.iter(self.readonly_session, self.inventory_index_id, GCP_TYPE_LIST, fetch_category=Categories.enabled_apis), self._convert_enabled_apis ) self.model_action_wrapper( DataAccess.iter(self.readonly_session, self.inventory_index_id, MEMBER_TYPE_LIST, with_parent=True), self._store_gsuite_membership, post_action=self._store_gsuite_membership_post ) self.model_action_wrapper( DataAccess.iter(self.readonly_session, self.inventory_index_id, GROUPS_SETTINGS_LIST), self._store_groups_settings ) self.dao.denorm_group_in_group(self.session) self.model_action_wrapper( DataAccess.iter(self.readonly_session, self.inventory_index_id, GCP_TYPE_LIST, fetch_category=Categories.iam_policy), self._store_iam_policy ) self.dao.expand_special_members(self.session) except Exception as e: # pylint: disable=broad-except LOGGER.exception(e) buf = StringIO() traceback.print_exc(file=buf) buf.seek(0) message = buf.read() LOGGER.debug('Importer has an exception: %s', message) self.model.set_error(message) else: LOGGER.debug('Set model status.') for row in inventory_index.warning_messages: self.model.add_warning('{}: {}'.format(row.resource_full_name, row.warning_message)) self.model.set_done(item_counter) finally: LOGGER.debug('Finished running importer.') self.session.commit() self.session.autocommit = autocommit self.session.autoflush = autoflush
# pylint: enable=too-many-statements
[docs] def model_action_wrapper(self, inventory_iterable, action, post_action=None, flush_count=1000, commit_count=50000): """Model action wrapper. This is used to reduce code duplication. Args: inventory_iterable (Iterable): Inventory iterable. action (func): Action taken during the iteration of the inventory list. post_action (func): Action taken after iterating the inventory list. flush_count (int): Flush every flush_count times. commit_count (int): Commit every commit_count times. Returns: int: Number of item iterated. """ LOGGER.debug('Performing model action: %s', action) idx = 0 for idx, inventory_data in enumerate(inventory_iterable, start=1): LOGGER.debug('Processing inventory data: %s', inventory_data) if isinstance(inventory_data, tuple): action(*inventory_data) else: action(inventory_data) if not idx % flush_count: # Flush database every flush_count resources LOGGER.debug('Flushing write session: %s.', idx) self._flush_session() if not idx % commit_count: LOGGER.debug('Committing write session: %s', idx) self._commit_session() if idx % flush_count: # Additional rows added since last flush. self._flush_session() if post_action: LOGGER.debug('Running post action: %s', post_action) post_action() LOGGER.debug('Committing model action: %s, with resource count: %s', action, idx) self._commit_session() return idx
[docs] def _store_gsuite_principal(self, principal): """Store a gsuite principal such as a group, user or member. Args: principal (object): object to store. Raises: Exception: if the principal type is unknown. """ gsuite_type = principal.get_resource_type() data = principal.get_resource_data() if gsuite_type == 'gsuite_user': member = 'user/{}'.format(data['primaryEmail'].lower()) elif gsuite_type == 'gsuite_group': member = 'group/{}'.format(data['email'].lower()) else: raise Exception('Unknown gsuite principal: {}'.format(gsuite_type)) if member not in self.member_cache: m_type, name = member.split('/', 1) self.member_cache[member] = self.dao.TBL_MEMBER( name=member, type=m_type, member_name=name) self.session.add(self.member_cache[member])
[docs] def _store_gsuite_membership_post(self): """Flush storing gsuite memberships.""" if not self.member_cache: return self.session.flush() # session.execute automatically flushes if self.membership_items: if get_sql_dialect(self.session) == 'sqlite': # SQLite doesn't support bulk insert for item in self.membership_items: stmt = self.dao.TBL_MEMBERSHIP.insert(item) self.session.execute(stmt) else: stmt = self.dao.TBL_MEMBERSHIP.insert( self.membership_items) self.session.execute(stmt)
[docs] def _store_gsuite_membership(self, child, parent): """Store a gsuite principal such as a group, user or member. Args: child (object): member item. parent (object): parent part of membership. """ def member_name(child): """Create the type:name representation for a non-group. Args: child (object): member to create representation from. Returns: str: type:name representation of the member. """ data = child.get_resource_data() return '{}/{}'.format(data['type'].lower(), data['email'].lower()) # Gsuite group members don't have to be part # of this domain, so we might see them for # the first time here. member = member_name(child) if member not in self.member_cache: m_type, name = member.split('/', 1) self.member_cache[member] = self.dao.TBL_MEMBER( name=member, type=m_type, member_name=name) self.session.add(self.member_cache[member]) parent_group = group_name(parent) if parent_group not in self.membership_map: self.membership_map[parent_group] = set() if member not in self.membership_map[parent_group]: self.membership_map[parent_group].add(member) self.membership_items.append( dict(group_name=group_name(parent), members_name=member))
[docs] def _store_groups_settings(self, settings): """Store gsuite settings. Args: settings (object): settings resource object. """ settings_dict = settings.get_resource_data() group_email = group_name(settings) if group_email not in self.groups_settings_cache: self.groups_settings_cache.add(group_email) settings_row = dict(group_name=group_email, settings=json.dumps(settings_dict, sort_keys=True)) stmt = self.dao.TBL_GROUPS_SETTINGS.insert(settings_row) self.session.execute(stmt)
[docs] def _store_iam_policy(self, policy): """Store the iam policy of the resource. Args: policy (object): IAM policy to store. Raises: KeyError: if member could not be found in any cache. """ bindings = policy.get_resource_data().get('bindings', []) policy_type_name = self._type_name(policy) for binding in bindings: role = binding['role'] if role not in self.role_cache: msg = 'Role reference in iam policy not found: {}'.format(role) self.model.add_warning(msg) continue # binding['members'] can have duplicate ids members = set(binding['members']) db_members = set() for member in members: member = member.replace(':', '/', 1).lower() # We still might hit external users or groups # that we haven't seen in gsuite. if member in self.member_cache and member not in db_members: db_members.add(self.member_cache[member]) continue if (member not in self.member_cache and member not in self.member_cache_policies): try: # This is the default case, e.g. 'group/foobar' m_type, name = member.split('/', 1) except ValueError: # Special groups like 'allUsers' done specify a type m_type, name = member, member self.member_cache_policies[member] = self.dao.TBL_MEMBER( name=member, type=m_type, member_name=name) self.session.add(self.member_cache_policies[member]) db_members.add(self.member_cache_policies[member]) binding_object = self.dao.TBL_BINDING( resource_type_name=policy_type_name, role_name=role, members=list(db_members)) self.session.add(binding_object) self._convert_iam_policy(policy)
[docs] def _store_resource(self, resource): """Store an inventory resource in the database. Args: resource (object): Resource object to convert from. """ handlers = { 'appengine_app': self._convert_gae_resource, 'appengine_instance': self._convert_gae_instance_resource, 'appengine_service': self._convert_gae_resource, 'appengine_version': self._convert_gae_resource, 'backendservice': self._convert_computeengine_resource, 'bigquery_table': self._convert_bigquery_table, 'billing_account': self._convert_billing_account, 'bucket': self._convert_bucket, 'cloudsqlinstance': self._convert_cloudsqlinstance, 'composite_root': self._convert_composite_root, 'compute_autoscaler': self._convert_computeengine_resource, 'compute_backendbucket': self._convert_computeengine_resource, 'compute_healthcheck': self._convert_computeengine_resource, 'compute_httphealthcheck': self._convert_computeengine_resource, 'compute_httpshealthcheck': self._convert_computeengine_resource, 'compute_license': self._convert_computeengine_resource, 'compute_project': self._convert_computeengine_resource, 'compute_router': self._convert_computeengine_resource, 'compute_sslcertificate': self._convert_computeengine_resource, 'compute_targethttpproxy': self._convert_computeengine_resource, 'compute_targethttpsproxy': self._convert_computeengine_resource, 'compute_targetinstance': self._convert_computeengine_resource, 'compute_targetpool': self._convert_computeengine_resource, 'compute_targetsslproxy': self._convert_computeengine_resource, 'compute_targettcpproxy': self._convert_computeengine_resource, 'compute_targetvpngateway': self._convert_computeengine_resource, 'compute_urlmap': self._convert_computeengine_resource, 'compute_vpntunnel': self._convert_computeengine_resource, 'crm_org_policy': self._convert_crm_org_policy, 'dataproc_cluster': self._convert_dataproc_cluster, 'dataset': self._convert_dataset, 'disk': self._convert_computeengine_resource, 'dns_managedzone': self._convert_clouddns_resource, 'dns_policy': self._convert_clouddns_resource, 'firewall': self._convert_computeengine_resource, 'folder': self._convert_folder, 'forwardingrule': self._convert_computeengine_resource, 'image': self._convert_computeengine_resource, 'instance': self._convert_computeengine_resource, 'instancegroup': self._convert_computeengine_resource, 'instancegroupmanager': self._convert_computeengine_resource, 'instancetemplate': self._convert_computeengine_resource, 'kms_cryptokey': self._convert_kms_resource, 'kms_cryptokeyversion': self._convert_kms_ckv_resource, 'kms_keyring': self._convert_kms_resource, 'kubernetes_cluster': self._convert_kubernetes_cluster, 'kubernetes_clusterrole': self._convert_kubernetes_clusterrole, 'kubernetes_clusterrolebinding': self._convert_kubernetes_binding, 'kubernetes_namespace': self._convert_kubernetes_namespace, 'kubernetes_node': self._convert_kubernetes_node, 'kubernetes_pod': self._convert_kubernetes_pod, 'kubernetes_role': self._convert_kubernetes_role, 'kubernetes_rolebinding': self._convert_kubernetes_rolebinding, 'lien': self._convert_lien, 'network': self._convert_computeengine_resource, 'organization': self._convert_organization, 'project': self._convert_project, 'pubsub_subscription': self._convert_pubsub_resource, 'pubsub_topic': self._convert_pubsub_resource, 'serviceaccount': self._convert_serviceaccount, 'serviceaccount_key': self._convert_serviceaccount_key, 'sink': self._convert_sink, 'snapshot': self._convert_computeengine_resource, 'spanner_database': self._convert_spanner_db_resource, 'spanner_instance': self._convert_spanner_resource, 'subnetwork': self._convert_computeengine_resource, None: None, } res_type = resource.get_resource_type() if resource else None handler = handlers.get(res_type) if handler: handler(resource) else: self.model.add_warning('No handler for type "{}"'.format(res_type))
[docs] def _convert_resource(self, resource, cached=False, display_key='name', email_key='email'): """Convert resource to a database object. Args: resource (Resource): A resource to store. cached (bool): Set to true for resources that have child resources or policies associated with them. display_key (str): The key in the resource dictionary to lookup to get the display name for the resource. email_key (str): The key in the resource dictionary to lookup to get the email associated with the resource. """ data = resource.get_resource_data() if self._is_root(resource): parent, type_name = None, self._type_name(resource) full_res_name = to_full_resource_name('', type_name) else: parent, full_res_name, type_name = self._full_resource_name( resource) row = self.dao.TBL_RESOURCE( cai_resource_name=resource.get_cai_resource_name(), cai_resource_type=resource.get_cai_resource_type(), full_name=full_res_name, type_name=type_name, name=resource.get_resource_id(), type=resource.get_resource_type(), display_name=data.get(display_key, ''), email=data.get(email_key, ''), data=resource.get_resource_data_raw(), parent=parent) self.session.add(row) if cached: self._add_to_cache(row, resource.id)
[docs] def _convert_billing_account(self, billing_account): """Convert a billing account to a database object. Args: billing_account (object): billing account to store. """ self._convert_resource(billing_account, cached=True, display_key='displayName')
[docs] def _convert_bucket(self, bucket): """Convert a bucket to a database object. Args: bucket (object): Bucket to store. """ self._convert_resource(bucket, cached=True)
[docs] def _convert_clouddns_resource(self, resource): """Convert a CloudDNS resource to a database object. Args: resource (dict): A resource to store. """ self._convert_resource(resource, cached=False)
[docs] def _convert_composite_root(self, resource): """Convert a composite root resource to a database object. Args: resource (dict): A resource to store. """ self._convert_resource(resource, cached=True)
[docs] def _convert_computeengine_resource(self, resource): """Convert an AppEngine resource to a database object. Args: resource (dict): An appengine resource to store. """ self._convert_resource(resource, cached=False)
[docs] def _convert_crm_org_policy(self, org_policy): """Convert an org policy to a database object. Args: org_policy (object): org policy to store. """ self._convert_resource(org_policy, cached=False, display_key='constraint')
[docs] def _convert_dataproc_cluster(self, cluster): """Convert a dataproc cluster to a database object. Args: cluster (object): Dataproc Cluster to store. """ self._convert_resource(cluster, cached=True, display_key='clusterName')
[docs] def _convert_dataset(self, dataset): """Convert a dataset to a database object. Args: dataset (object): Dataset to store. """ self._convert_resource(dataset, cached=True)
[docs] def _convert_folder(self, folder): """Convert a folder to a database object. Args: folder (object): Folder to store. """ self._convert_resource(folder, cached=True, display_key='displayName')
[docs] def _convert_gae_instance_resource(self, resource): """Convert an AppEngine Instance resource to a database object. Args: resource (dict): A resource to store. """ self._convert_resource(resource, cached=False)
[docs] def _convert_gae_resource(self, resource): """Convert an AppEngine resource to a database object. Args: resource (dict): A resource to store. """ self._convert_resource(resource, cached=True)
[docs] def _convert_kms_ckv_resource(self, resource): """Convert a KMS CryptoKeyVersion resource to a database object. Args: resource (dict): A resource to store. """ # No child resources, so do not cache. self._convert_resource(resource, cached=False, display_key='name')
[docs] def _convert_kms_resource(self, resource): """Convert a KMS resource to a database object. Args: resource (dict): A resource to store. """ self._convert_resource(resource, cached=True, display_key='name')
[docs] def _convert_kubernetes_cluster(self, kubernetes_cluster): """Convert a Kubernetes Cluster resource to a database object. Args: kubernetes_cluster (dict): A Kubernetes cluster resource to store. """ self._convert_resource(kubernetes_cluster, cached=True, display_key='kubernetesClusterName')
[docs] def _convert_kubernetes_clusterrole(self, kubernetes_clusterrole): """Convert a Kubernetes ClusterRole resource to a database object. Args: kubernetes_clusterrole (dict): A Kubernetes ClusterRole resource to store. """ self._convert_resource(kubernetes_clusterrole, cached=False, display_key='kubernetesClusterRole')
[docs] def _convert_kubernetes_binding(self, kubernetes_clusterrolebinding): """Convert a Kubernetes ClusterRoleBinding resource to a database object. Args: kubernetes_clusterrolebinding (dict): A Kubernetes ClusterRoleBinding resource to store. """ self._convert_resource(kubernetes_clusterrolebinding, cached=False, display_key='kubernetesClusterRoleBinding')
[docs] def _convert_kubernetes_namespace(self, kubernetes_namespace): """Convert a Kubernetes Namespace resource to a database object. Args: kubernetes_namespace (dict): A Kubernetes Namespace resource to store. """ self._convert_resource(kubernetes_namespace, cached=True, display_key='kubernetesNamespace')
[docs] def _convert_kubernetes_node(self, kubernetes_node): """Convert a Kubernetes Node resource to a database object. Args: kubernetes_node (dict): A Kubernetes Node resource to store. """ self._convert_resource(kubernetes_node, cached=False, display_key='kubernetesNode')
[docs] def _convert_kubernetes_pod(self, kubernetes_pod): """Convert a Kubernetes Pod resource to a database object. Args: kubernetes_pod (dict): A Kubernetes Pod resource to store. """ self._convert_resource(kubernetes_pod, cached=False, display_key='kubernetesPod')
[docs] def _convert_kubernetes_role(self, kubernetes_role): """Convert a Kubernetes Role resource to a database object. Args: kubernetes_role (dict): A Kubernetes Role resource to store. """ self._convert_resource(kubernetes_role, cached=False, display_key='kubernetesRole')
[docs] def _convert_kubernetes_rolebinding(self, kubernetes_rolebinding): """Convert a Kubernetes RoleBinding resource to a database object. Args: kubernetes_rolebinding (dict): A Kubernetes RoleBinding resource to store. """ self._convert_resource(kubernetes_rolebinding, cached=False, display_key='kubernetesRoleBinding')
[docs] def _convert_lien(self, lien): """Convert a lien to a database object. Args: lien (object): Lien to store. """ self._convert_resource(lien, cached=True)
[docs] def _convert_organization(self, organization): """Convert an organization a database object. Args: organization (object): Organization to store. """ self._convert_resource(organization, cached=True, display_key='displayName')
[docs] def _convert_pubsub_resource(self, resource): """Convert a PubSub resource to a database object. Args: resource (object): Pubsub resource to store. """ self._convert_resource(resource, cached=True)
[docs] def _convert_project(self, project): """Convert a project to a database object. Args: project (object): Project to store. """ self._convert_resource(project, cached=True)
[docs] def _convert_serviceaccount(self, service_account): """Convert a service account to a database object. Args: service_account (object): Service account to store. """ self._convert_resource(service_account, cached=True, display_key='displayName', email_key='email')
[docs] def _convert_serviceaccount_key(self, service_account_key): """Convert a service account key to a database object. Args: service_account_key (object): Service account key to store. """ self._convert_resource(service_account_key, cached=False)
[docs] def _convert_sink(self, sink): """Convert a log sink to a database object. Args: sink (object): Sink to store. """ self._convert_resource(sink, cached=False, email_key='writerIdentity')
[docs] def _convert_spanner_db_resource(self, resource): """Convert a Spanner Database resource to a database object. Args: resource (dict): A resource to store. """ self._convert_resource(resource, cached=False)
[docs] def _convert_spanner_resource(self, resource): """Convert a Spanner Instance resource to a database object. Args: resource (dict): A resource to store. """ self._convert_resource(resource, cached=True, display_key='displayName')
# The following methods require more complex logic than _convert_resource # provides.
[docs] def _convert_cloudsqlinstance(self, cloudsqlinstance): """Convert a cloudsqlinstance to a database object. Args: cloudsqlinstance (object): Cloudsql to store. """ data = cloudsqlinstance.get_resource_data() parent, full_res_name, type_name = self._full_resource_name( cloudsqlinstance) parent_key = get_resource_id_from_type_name(parent.type_name) resource_identifier = '{}:{}'.format(parent_key, cloudsqlinstance.get_resource_id()) type_name = to_type_name(cloudsqlinstance.get_resource_type(), resource_identifier) resource = self.dao.TBL_RESOURCE( cai_resource_name=cloudsqlinstance.get_cai_resource_name(), cai_resource_type=cloudsqlinstance.get_cai_resource_type(), full_name=full_res_name, type_name=type_name, name=cloudsqlinstance.get_resource_id(), type=cloudsqlinstance.get_resource_type(), display_name=data.get('name', ''), email=data.get('email', ''), data=cloudsqlinstance.get_resource_data_raw(), parent=parent) self.session.add(resource)
[docs] def _convert_dataset_policy(self, dataset_policy): """Convert a dataset policy to a database object. Args: dataset_policy (object): Dataset policy to store. """ # TODO: Dataset policies should be integrated in the model, not stored # as a resource. parent, full_res_name = self._get_parent(dataset_policy) policy_type_name = to_type_name( dataset_policy.get_category(), dataset_policy.get_resource_id()) policy_res_name = to_full_resource_name(full_res_name, policy_type_name) resource = self.dao.TBL_RESOURCE( cai_resource_name=dataset_policy.get_cai_resource_name(), cai_resource_type=dataset_policy.get_cai_resource_type(), full_name=policy_res_name, type_name=policy_type_name, name=dataset_policy.get_resource_id(), type=dataset_policy.get_category(), data=dataset_policy.get_resource_data_raw(), parent=parent) self.session.add(resource)
[docs] def _convert_enabled_apis(self, enabled_apis): """Convert a description of enabled APIs to a database object. Args: enabled_apis (object): Enabled APIs description to store. """ parent, full_res_name = self._get_parent(enabled_apis) apis_type_name = to_type_name( enabled_apis.get_category(), ':'.join(parent.type_name.split('/'))) apis_res_name = to_full_resource_name(full_res_name, apis_type_name) resource = self.dao.TBL_RESOURCE( cai_resource_name=enabled_apis.get_cai_resource_name(), cai_resource_type=enabled_apis.get_cai_resource_type(), full_name=apis_res_name, type_name=apis_type_name, name=enabled_apis.get_resource_id(), type=enabled_apis.get_category(), data=enabled_apis.get_resource_data_raw(), parent=parent) self.session.add(resource)
[docs] def _convert_gcs_policy(self, gcs_policy): """Convert a gcs policy to a database object. Args: gcs_policy (object): Cloud Storage Bucket ACL policy to store. """ parent, full_res_name = self._get_parent(gcs_policy) policy_type_name = to_type_name( gcs_policy.get_category(), gcs_policy.get_resource_id()) policy_res_name = to_full_resource_name(full_res_name, policy_type_name) resource = self.dao.TBL_RESOURCE( cai_resource_name=gcs_policy.get_cai_resource_name(), cai_resource_type=gcs_policy.get_cai_resource_type(), full_name=policy_res_name, type_name=policy_type_name, name=gcs_policy.get_resource_id(), type=gcs_policy.get_category(), data=gcs_policy.get_resource_data_raw(), parent=parent) self.session.add(resource)
[docs] def _convert_iam_policy(self, iam_policy): """Convert an IAM policy to a database object. Args: iam_policy (object): IAM policy to store. """ _, full_res_name = self._get_parent(iam_policy) parent_type_name = self._type_name(iam_policy) iam_policy_type_name = to_type_name( iam_policy.get_category(), ':'.join(parent_type_name.split('/'))) iam_policy_full_res_name = to_full_resource_name( full_res_name, iam_policy_type_name) resource = self.dao.TBL_RESOURCE( cai_resource_name=iam_policy.get_cai_resource_name(), cai_resource_type=iam_policy.get_cai_resource_type(), full_name=iam_policy_full_res_name, type_name=iam_policy_type_name, name=iam_policy.get_resource_id(), type=iam_policy.get_category(), data=iam_policy.get_resource_data_raw(), parent_type_name=parent_type_name) self.session.add(resource)
[docs] def _convert_role(self, role): """Convert a role to a database object. Args: role (object): Role to store. """ data = role.get_resource_data() role_name = data.get('name') LOGGER.debug('Converting role: %s', role_name) LOGGER.debug('role data: %s', data) if role_name in self.role_cache: LOGGER.warning('Duplicate role_name: %s', role_name) return is_custom = not role_name.startswith('roles/') db_permissions = [] if 'includedPermissions' not in data: self.model.add_warning( 'Role missing permissions: {}'.format( data.get('name', '<missing name>'))) else: for perm_name in data['includedPermissions']: if perm_name not in self.permission_cache: permission = self.dao.TBL_PERMISSION( name=perm_name) self.permission_cache[perm_name] = permission self.session.add(permission) db_permissions.append(self.permission_cache[perm_name]) dbrole = self.dao.TBL_ROLE( name=role_name, title=data.get('title', ''), stage=data.get('stage', ''), description=data.get('description', ''), custom=is_custom, permissions=db_permissions) self.role_cache[data['name']] = dbrole self.session.add(dbrole) LOGGER.debug('Adding role %s to session', role_name) if is_custom: parent, full_res_name, type_name = self._full_resource_name(role) role_resource = self.dao.TBL_RESOURCE( cai_resource_name=role.get_cai_resource_name(), cai_resource_type=role.get_cai_resource_type(), full_name=full_res_name, type_name=type_name, name=role.get_resource_id(), type=role.get_resource_type(), display_name=data.get('title'), data=role.get_resource_data_raw(), parent=parent) self._add_to_cache(role_resource, role.id) self.session.add(role_resource) LOGGER.debug('Adding role resource :%s to session', role_name) LOGGER.debug('Role resource :%s', role_resource)
[docs] def _convert_role_post(self): """Executed after all roles were handled. Performs bulk insert.""" self.session.add_all(list(self.permission_cache.values())) self.session.add_all(list(self.role_cache.values()))
[docs] def _convert_service_config(self, service_config): """Convert Kubernetes Service Config to a database object. Args: service_config (dict): A Service Config resource to store. """ parent, full_res_name = self._get_parent(service_config) sc_type_name = to_type_name( service_config.get_category(), parent.type_name) sc_res_name = to_full_resource_name(full_res_name, sc_type_name) resource = self.dao.TBL_RESOURCE( cai_resource_name=service_config.get_cai_resource_name(), cai_resource_type=service_config.get_cai_resource_type(), full_name=sc_res_name, type_name=sc_type_name, name=service_config.get_resource_id(), type=service_config.get_category(), data=service_config.get_resource_data_raw(), parent=parent) self.session.add(resource)
[docs] def _convert_bigquery_table(self, table): """Convert a table to a database object. Args: table (object): table to store. """ self._convert_resource(table, cached=True)
[docs] def _add_to_cache(self, resource, resource_id): """Add a resource to the cache for parent lookup. Args: resource (object): Resource to put in the cache. resource_id (int): The database key for the resource. """ full_res_name = resource.full_name self.resource_cache[resource_id] = (resource, full_res_name)
[docs] def _full_resource_name(self, resource): """Returns the parent object, full resource name and type name. Args: resource (object): Resource whose full resource name and parent should be returned. Returns: str: full resource name for the provided resource. """ type_name = self._type_name(resource) parent, full_res_name = self._get_parent(resource) full_resource_name = to_full_resource_name(full_res_name, type_name) return parent, full_resource_name, type_name
[docs] def _get_parent(self, resource): """Return the parent object for a resource from cache. Args: resource (object): Resource whose parent to look for. Returns: tuple: cached object and full resource name """ parent_id = resource.get_parent_id() return self.resource_cache[parent_id]
[docs] def _is_root(self, resource): """Checks if the resource is an inventory root. Result is cached. Args: resource (object): Resource to check. Returns: bool: Whether the resource is root or not """ if not self.found_root: is_root = not resource.get_parent_id() if is_root: self.found_root = True return is_root return False
[docs] @staticmethod def _type_name(resource): """Return the type/name for that resource. This is a simple wrapper for the to_type_name function. Args: resource (object): Resource to retrieve type/name for. Returns: str: type/name representation of the resource. """ return to_type_name( resource.get_resource_type(), resource.get_resource_id())
[docs]def group_name(group): """Create the type:name representation for a group. Args: group (object): group to create representation from. Returns: str: group:name representation of the group. """ data = group.get_resource_data() return 'group/{}'.format(data['email'].lower())
[docs]def by_source(source): """Helper to resolve client provided import sources. Args: source (str): Source to import from. Returns: Importer: Chosen by source. """ return { 'INVENTORY': InventoryImporter, 'EMPTY': EmptyImporter, }[source.upper()]