google.cloud.forseti.services.inventory.storage module

Inventory storage implementation.

class Categories[source]

Bases: enum.Enum

Inventory Categories.

billing_info = 5
dataset_policy = 4
enabled_apis = 6
gcs_policy = 3
iam_policy = 2
kubernetes_service_config = 7
resource = 1
class DataAccess[source]

Bases: object

Access to inventory for services.

classmethod delete(session, inventory_index_id)[source]

Delete an inventory index entry by id.

Parameters:
  • session (object) – Database session.
  • inventory_index_id (str) – Id specifying which inventory to delete.
Returns:

An expunged entry corresponding the inventory_index_id.

Return type:

InventoryIndex

Raises:

Exception – Reraises any exception.

classmethod get(session, inventory_index_id)[source]

Get an inventory index entry by id.

Parameters:
  • session (object) – Database session.
  • inventory_index_id (str) – Inventory id.
Returns:

Entry corresponding the id

Return type:

InventoryIndex

classmethod get_inventory_index_id_by_scanner_index_id(session, scanner_index_id)[source]

List all inventory index entries.

Parameters:
  • session (object) – Database session.
  • scanner_index_id (int) – id of the scanner in scanner_index table
Returns:

inventory index id

Return type:

int64

classmethod get_inventory_indexes_older_than_cutoff(session, cutoff_datetime)[source]

Get all inventory index entries older than the cutoff.

Parameters:
  • session (object) – Database session.
  • cutoff_datetime (datetime) – The cutoff point to find any older inventory index entries.
Returns:

InventoryIndex

Return type:

list

classmethod get_latest_inventory_index_id(session)[source]

List all inventory index entries.

Parameters:session (object) – Database session.
Returns:inventory index id
Return type:int64
classmethod get_root(session, inventory_index_id)[source]

Get the resource root from the inventory.

Parameters:
  • session (object) – Database session.
  • inventory_index_id (str) – the id of the inventory to query.
Returns:

A row in gcp_inventory of the root

Return type:

object

classmethod iter(session, inventory_index_id, type_list=None, fetch_category=<Categories.resource: 1>, with_parent=False)[source]

Iterate the objects in the storage.

Parameters:
  • session (object) – Database session.
  • inventory_index_id (str) – the id of the inventory to open.
  • type_list (list) – List of types to iterate over, or [] for all.
  • fetch_category (Categories) – The category of data to fetch.
  • with_parent (bool) – Join parent with results, yield tuples.
Yields:

object – Single row object or child/parent if ‘with_parent’ is set.

classmethod list(session)[source]

List all inventory index entries.

Parameters:session (object) – Database session.
Yields:InventoryIndex – Generates each row
classmethod type_exists(session, inventory_index_id, type_list=None)[source]

Check if certain types of resources exists in the inventory.

Parameters:
  • session (object) – Database session.
  • inventory_index_id (str) – the id of the inventory to query.
  • type_list (list) – List of types to check.
Returns:

If these types of resources exists.

Return type:

bool

class Inventory(**kwargs)[source]

Bases: sqlalchemy.ext.declarative.api.Base

Resource inventory table.

__repr__()[source]

String representation of the database row object.

Returns:A description of inventory_index
Return type:str
classmethod _get_policy_full_name(resource, policy_name)[source]

Create a full name for a resource policy.

Parameters:
  • resource (Resource) – Crawled resource.
  • policy_name (str) – The category name for the policy data.
Returns:

A full name for the policy.

Return type:

str

_sa_class_manager = {'cai_resource_name': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae70737d8>, 'cai_resource_type': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae7073888>, 'category': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae7073938>, 'full_name': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae7073728>, 'id': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae7073468>, 'inventory_errors': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae7073d58>, 'inventory_index_id': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae7073678>, 'other': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae7073ca8>, 'parent_id': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae7073bf8>, 'resource_data': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae7073b48>, 'resource_id': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae7073a98>, 'resource_type': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae70739e8>}
cai_resource_name
cai_resource_type
category
classmethod from_resource(index, resource)[source]

Creates a database row object from a crawled resource.

Parameters:
Returns:

A tuple containing a single row for the main

resource, and a list of rows for any additional policies attached to the resource.

Return type:

Tuple[dict, list]

full_name
get_cai_resource_name()[source]

Get the row’s cai resource name.

Returns:cai resource name.
Return type:str
get_cai_resource_type()[source]

Get the row’s cai resource type.

Returns:cai resource type.
Return type:str
get_category()[source]

Get the row’s data category.

Returns:data category.
Return type:str
get_full_name()[source]

Get the row’s full name.

Returns:resource full name.
Return type:str
get_inventory_errors()[source]

Get the row’s error data.

Returns:row’s error data.
Return type:str
get_other()[source]

Get the row’s other data.

Returns:row’s other data.
Return type:dict
get_parent_id()[source]

Get the row’s parent id.

Returns:parent id.
Return type:int
get_resource_data()[source]

Get the row’s metadata.

Returns:row’s metadata.
Return type:dict
get_resource_data_raw()[source]

Get the row’s data json string.

Returns:row’s raw data.
Return type:str
get_resource_id()[source]

Get the row’s resource id.

Returns:resource id.
Return type:str
get_resource_type()[source]

Get the row’s resource type.

Returns:resource type.
Return type:str
static get_schema_update_actions()[source]

Maintain all the schema changes for this table.

Returns:A mapping of Action: Column.
Return type:dict
id
inventory_errors
inventory_index_id
other
parent_id
resource_data
resource_id
resource_type
class InventoryIndex(**kwargs)[source]

Bases: sqlalchemy.ext.declarative.api.Base

Represents a GCP inventory.

__repr__()[source]

Object string representation.

Returns:String representation of the object.
Return type:str
_sa_class_manager = {'completed_at_datetime': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae705beb8>, 'counter': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae70731a8>, 'created_at_datetime': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae705be08>, 'id': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae705bd58>, 'inventory_index_errors': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae7073308>, 'inventory_index_warnings': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae7073258>, 'inventory_status': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae705bf68>, 'message': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae70733b8>, 'progress': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae70730f8>, 'schema_version': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae7073048>, 'warning_count': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae705bc50>, 'warning_messages': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae705bca8>}
add_warning(engine, resource_full_name, warning)[source]

Add a warning to the inventory_warnings table.

Parameters:
  • engine (sqlalchemy.engine.Engine) – Engine to write to.
  • resource_full_name (str) – The full name of the resource that raised the error.
  • warning (str) – Warning message
complete(status='SUCCESS')[source]

Mark the inventory as completed with a final inventory_status.

Parameters:status (str) – Final inventory_status.
completed_at_datetime
counter
classmethod create()[source]

Create a new inventory index row.

Returns:InventoryIndex row object.
Return type:InventoryIndex
created_at_datetime
get_details(session)[source]

Generate/return inventory details for this inventory index.

Includes delete pending/active resource types and hidden/shown datasets.

Parameters:session (object) – session object to work on.
Returns:a (resource type -> count) dictionary
Return type:dict
get_hidden_resource_details(session, resource_type)[source]

Count of the hidden and shown specified resources.

Generate/return the count of hidden resources (e.g. dataset) for this inventory index.

Parameters:
  • session (object) – session object to work on.
  • resource_type (str) – resource type to find details for.
Returns:

a (hidden_resource -> count) dictionary

Return type:

dict

get_lifecycle_state_details(session, resource_type_input)[source]

Count of lifecycle states of the specified resources.

Generate/return the count of lifecycle states (ACTIVE, DELETE_PENDING) of the specific resource type input (project, folder) for this inventory index.

Parameters:
  • session (object) – session object to work on.
  • resource_type_input (str) – resource type to get lifecycle states.
Returns:

a (lifecycle state -> count) dictionary

Return type:

dict

get_summary(session)[source]

Generate/return an inventory summary for this inventory index.

Parameters:session (object) – session object to work on.
Returns:a (resource type -> count) dictionary
Return type:dict
id
inventory_index_errors
inventory_index_warnings
inventory_status
message
progress
schema_version
set_error(message)[source]

Indicate a broken import.

Parameters:message (str) – Error message to set.
warning_count
warning_messages
class InventoryWarnings(**kwargs)[source]

Bases: sqlalchemy.ext.declarative.api.Base

Warning messages generated during the creation of the inventory.

_sa_class_manager = {'id': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae705b620>, 'inventory_index_id': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae705b780>, 'resource_full_name': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae705b888>, 'warning_message': <sqlalchemy.orm.attributes.InstrumentedAttribute object at 0x7f4ae705b938>}
id
inventory_index_id
resource_full_name
warning_message
class Storage(session, engine)[source]

Bases: google.cloud.forseti.services.inventory.base.storage.Storage

Inventory storage used during creation.

__enter__()[source]

To support with statement for auto closing.

Returns:The inventory storage object
Return type:Storage
__exit__(type_p, value, traceback)[source]

To support with statement for auto closing.

Parameters:
  • type_p (object) – Unused.
  • value (object) – Unused.
  • traceback (object) – Unused.
_create()[source]

Create a new inventory.

Returns:Index number of the created inventory.
Return type:int
Raises:Exception – Reraises any exception.
_open(inventory_index_id)[source]

Open an existing inventory.

Parameters:inventory_index_id (str) – the id of the inventory to open.
Returns:The inventory index db row.
Return type:object
_require_opened()[source]

Make sure the storage is in ‘open’ state.

Raises:Exception – If storage is not opened.
close()[source]

Close the storage.

Raises:Exception – If the storage was not opened before or if the storage is writeable but neither rollback nor commit has been called.
commit()[source]

Commit the stored inventory.

error(message)[source]

Store a fatal error in storage. This will help debug problems.

Parameters:message (str) – Error message describing the problem.
open(handle=None)[source]

Open the storage, potentially create a new index.

Parameters:handle (str) – If None, create a new index instead of opening an existing one.
Returns:Index id of the opened or created inventory.
Return type:str
Raises:Exception – if open was called more than once
rollback()[source]

Roll back the stored inventory, but keep the index entry.

warning(resource_full_name, message)[source]

Store a Warning message in storage. This will help debug problems.

Parameters:
  • resource_full_name (str) – The full name of the resource that raised the error.
  • message (str) – Warning message describing the problem.
write(resource)[source]

Write a resource to the storage and updates its row

Parameters:resource (object) – Resource object to store in db.
initialize(engine)[source]

Create all tables in the database if not existing.

Parameters:engine (object) – Database engine to operate on.