Documentation for Google Cloud Data Loss Prevention (#8201) (#9651)

apache · Aug 27, 2020 · 91ff31a · 91ff31a
1 parent bb3e45c
commit 91ff31a
Show file tree

Hide file tree

Showing 4 changed files with 542 additions and 1 deletion.
diff --git a/airflow/providers/google/cloud/example_dags/example_dlp.py b/airflow/providers/google/cloud/example_dags/example_dlp.py
@@ -31,8 +31,14 @@
 from airflow import models
 from airflow.providers.google.cloud.operators.dlp import (
  CloudDLPCreateInspectTemplateOperator,
+ CloudDLPCreateJobTriggerOperator,
+ CloudDLPCreateStoredInfoTypeOperator,
  CloudDLPDeleteInspectTemplateOperator,
+ CloudDLPDeleteJobTriggerOperator,
+ CloudDLPDeleteStoredInfoTypeOperator,
  CloudDLPInspectContentOperator,
+ CloudDLPUpdateJobTriggerOperator,
+ CloudDLPUpdateStoredInfoTypeOperator,
 )
 from airflow.utils.dates import days_ago
 
@@ -54,6 +60,7 @@
  start_date=days_ago(1),
  tags=['example'],
 ) as dag:
+ # [START howto_operator_dlp_create_inspect_template]
  create_template = CloudDLPCreateInspectTemplateOperator(
  project_id=GCP_PROJECT,
  inspect_template=INSPECT_TEMPLATE,
@@ -62,17 +69,104 @@
  do_xcom_push=True,
  dag=dag,
  )
+ # [END howto_operator_dlp_create_inspect_template]
 
+ # [START howto_operator_dlp_use_inspect_template]
  inspect_content = CloudDLPInspectContentOperator(
  task_id="inpsect_content",
  project_id=GCP_PROJECT,
  item=ITEM,
  inspect_template_name="{{ task_instance.xcom_pull('create_template', key='return_value')['name'] }}",
  dag=dag,
  )
+ # [END howto_operator_dlp_use_inspect_template]
 
+ # [START howto_operator_dlp_delete_inspect_template]
  delete_template = CloudDLPDeleteInspectTemplateOperator(
  task_id="delete_template", template_id=TEMPLATE_ID, project_id=GCP_PROJECT, dag=dag,
  )
+ # [END howto_operator_dlp_delete_inspect_template]
 
  create_template >> inspect_content >> delete_template
+
+
+CUSTOM_INFO_TYPES = [{"info_type": {"name": "C_MRN"}, "regex": {"pattern": "[1-9]{3}-[1-9]{1}-[1-9]{5}"},}]
+CUSTOM_INFO_TYPE_ID = "custom_info_type"
+UPDATE_CUSTOM_INFO_TYPE = [
+ {"info_type": {"name": "C_MRN"}, "regex": {"pattern": "[a-z]{3}-[a-z]{1}-[a-z]{5}"},}
+]
+
+with models.DAG(
+ "example_gcp_dlp_info_types",
+ schedule_interval=None,
+ start_date=days_ago(1),
+ tags=["example", "dlp", "info-types"],
+) as dag:
+ # [START howto_operator_dlp_create_info_type]
+ create_info_type = CloudDLPCreateStoredInfoTypeOperator(
+ project_id=GCP_PROJECT,
+ config=CUSTOM_INFO_TYPES,
+ stored_info_type_id=CUSTOM_INFO_TYPE_ID,
+ dag=dag,
+ task_id="create_info_type",
+ )
+ # [END howto_operator_dlp_create_info_type]
+ # [START howto_operator_dlp_update_info_type]
+ update_info_type = CloudDLPUpdateStoredInfoTypeOperator(
+ project_id=GCP_PROJECT,
+ stored_info_type_id=CUSTOM_INFO_TYPE_ID,
+ config=UPDATE_CUSTOM_INFO_TYPE,
+ dag=dag,
+ task_id="update_info_type",
+ )
+ # [END howto_operator_dlp_update_info_type]
+ # [START howto_operator_dlp_delete_info_type]
+ delete_info_type = CloudDLPDeleteStoredInfoTypeOperator(
+ project_id=GCP_PROJECT, stored_info_type_id=CUSTOM_INFO_TYPE_ID, dag=dag, task_id="delete_info_type",
+ )
+ # [END howto_operator_dlp_delete_info_type]
+ create_info_type >> update_info_type >> delete_info_type
+
+SCHEDULE = {"recurrence_period_duration": {"seconds": 60 * 60 * 24}}
+JOB = {
+ "inspect_config": INSPECT_CONFIG,
+}
+
+JOB_TRIGGER = {
+ "inspect_job": JOB,
+ "triggers": [{"schedule": SCHEDULE}],
+ "status": "HEALTHY",
+}
+
+TRIGGER_ID = "example_trigger"
+
+with models.DAG(
+ "example_gcp_dlp_job", schedule_interval=None, start_date=days_ago(1), tags=["example", "dlp_job"],
+) as dag: # [START howto_operator_dlp_create_job_trigger]
+ create_trigger = CloudDLPCreateJobTriggerOperator(
+ project_id=GCP_PROJECT,
+ job_trigger=JOB_TRIGGER,
+ trigger_id=TRIGGER_ID,
+ dag=dag,
+ task_id="create_trigger",
+ )
+ # [END howto_operator_dlp_create_job_trigger]
+ UPDATED_SCHEDULE = {"recurrence_period_duration": {"seconds": 2 * 60 * 60 * 24}}
+
+ JOB_TRIGGER["triggers"] = [{"schedule": UPDATED_SCHEDULE}]
+
+ # [START howto_operator_dlp_update_job_trigger]
+ update_trigger = CloudDLPUpdateJobTriggerOperator(
+ project_id=GCP_PROJECT,
+ job_trigger_id=TRIGGER_ID,
+ job_trigger=JOB_TRIGGER,
+ dag=dag,
+ task_id="update_info_type",
+ )
+ # [END howto_operator_dlp_update_job_trigger]
+ # [START howto_operator_dlp_delete_job_trigger]
+ delete_trigger = CloudDLPDeleteJobTriggerOperator(
+ project_id=GCP_PROJECT, job_trigger_id=TRIGGER_ID, dag=dag, task_id="delete_info_type",
+ )
+ # [END howto_operator_dlp_delete_job_trigger]
+ create_trigger >> update_trigger >> delete_trigger