Split Display Video 360 example into smaler DAGs (#10077)

apache · Jul 31, 2020 · 4c84661 · 4c84661
1 parent 03c4351
commit 4c84661
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 4 deletions.
diff --git a/airflow/providers/google/marketing_platform/example_dags/example_display_video.py b/airflow/providers/google/marketing_platform/example_dags/example_display_video.py
@@ -84,7 +84,7 @@
  "example_display_video",
  schedule_interval=None, # Override to match your needs,
  start_date=dates.days_ago(1)
-) as dag:
+) as dag1:
  # [START howto_google_display_video_createquery_report_operator]
  create_report = GoogleDisplayVideo360CreateReportOperator(
  body=REPORT, task_id="create_report"
@@ -119,14 +119,21 @@
  )
  # [END howto_google_display_video_deletequery_report_operator]
 
+ create_report >> run_report >> wait_for_report >> get_report >> delete_report
+
+with models.DAG(
+ "example_display_video_misc",
+ schedule_interval=None, # Override to match your needs,
+ start_date=dates.days_ago(1)
+) as dag2:
  # [START howto_google_display_video_upload_multiple_entity_read_files_to_big_query]
  upload_erf_to_bq = GCSToBigQueryOperator(
  task_id='upload_erf_to_bq',
  bucket=BUCKET,
  source_objects=ERF_SOURCE_OBJECT,
  destination_project_dataset_table=f"{BQ_DATA_SET}.gcs_to_bq_table",
  write_disposition='WRITE_TRUNCATE',
-  dag=dag)
+ )
  # [END howto_google_display_video_upload_multiple_entity_read_files_to_big_query]
 
  # [START howto_google_display_video_download_line_items_operator]
@@ -147,6 +154,11 @@
  )
  # [END howto_google_display_video_upload_line_items_operator]
 
+with models.DAG(
+ "example_display_video_sdf",
+ schedule_interval=None, # Override to match your needs,
+ start_date=dates.days_ago(1)
+) as dag3:
  # [START howto_google_display_video_create_sdf_download_task_operator]
  create_sdf_download_task = GoogleDisplayVideo360CreateSDFDownloadTaskOperator(
  task_id="create_sdf_download_task", body_request=CREATE_SDF_DOWNLOAD_TASK_BODY_REQUEST
@@ -181,9 +193,7 @@
  {"name": "post_abbr", "type": "STRING", "mode": "NULLABLE"},
  ],
  write_disposition="WRITE_TRUNCATE",
- dag=dag,
  )
  # [END howto_google_display_video_gcs_to_big_query_operator]
 
- create_report >> run_report >> wait_for_report >> get_report >> delete_report
  create_sdf_download_task >> wait_for_operation >> save_sdf_in_gcs >> upload_sdf_to_big_query
diff --git a/tests/providers/google/marketing_platform/operators/test_display_video_system.py b/tests/providers/google/marketing_platform/operators/test_display_video_system.py
@@ -47,3 +47,11 @@ def tearDown(self):
  @provide_gcp_context(GMP_KEY, scopes=SCOPES)
  def test_run_example_dag(self):
  self.run_dag('example_display_video', MARKETING_DAG_FOLDER)
+
+ @provide_gcp_context(GMP_KEY, scopes=SCOPES)
+ def test_run_example_dag_misc(self):
+ self.run_dag('example_display_video_misc', MARKETING_DAG_FOLDER)
+
+ @provide_gcp_context(GMP_KEY, scopes=SCOPES)
+ def test_run_example_dag_sdf(self):
+ self.run_dag('example_display_video_sdf', MARKETING_DAG_FOLDER)