Migrate Google sheets example to new design AIP-47 (#24975)

related: #22447, #22430
apache · Jul 12, 2022 · a13c51e · a13c51e
1 parent a038b52
commit a13c51e
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 59 deletions.
diff --git a/docs/apache-airflow-providers-google/operators/suite/sheets.rst b/docs/apache-airflow-providers-google/operators/suite/sheets.rst
@@ -44,7 +44,7 @@ Create spreadsheet
 To create new spreadsheet you can use the
 :class:`~airflow.providers.google.suite.operators.sheets.GoogleSheetsCreateSpreadsheetOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/suite/example_dags/example_sheets.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_sheets.py
  :language: python
  :dedent: 4
  :start-after: [START create_spreadsheet]
@@ -55,7 +55,7 @@ You can use :ref:`Jinja templating <concepts:jinja-templating>` with
 
 To get the URL of newly created spreadsheet use XCom value:
 
-.. exampleinclude:: /../../airflow/providers/google/suite/example_dags/example_sheets.py
+.. exampleinclude:: /../../tests/system/providers/google/cloud/gcs/example_sheets.py
  :language: python
  :dedent: 4
  :start-after: [START print_spreadsheet_url]

diff --git a/tests/providers/google/suite/operators/test_sheets_system.py b/tests/providers/google/suite/operators/test_sheets_system.py
diff --git a/...ogle/suite/example_dags/example_sheets.py → ...viders/google/cloud/gcs/example_sheets.py b/...ogle/suite/example_dags/example_sheets.py → ...viders/google/cloud/gcs/example_sheets.py
@@ -21,11 +21,17 @@
 
 from airflow import models
 from airflow.operators.bash import BashOperator
+from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator
 from airflow.providers.google.cloud.transfers.sheets_to_gcs import GoogleSheetsToGCSOperator
 from airflow.providers.google.suite.operators.sheets import GoogleSheetsCreateSpreadsheetOperator
 from airflow.providers.google.suite.transfers.gcs_to_sheets import GCSToGoogleSheetsOperator
+from airflow.utils.trigger_rule import TriggerRule
 
-GCS_BUCKET = os.environ.get("SHEETS_GCS_BUCKET", "test28397ye")
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT")
+DAG_ID = "example_sheets_gcs"
+
+BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}"
 SPREADSHEET_ID = os.environ.get("SPREADSHEET_ID", "1234567890qwerty")
 NEW_SPREADSHEET_ID = os.environ.get("NEW_SPREADSHEET_ID", "1234567890qwerty")
 
@@ -35,16 +41,20 @@
 }
 
 with models.DAG(
- "example_sheets_gcs",
+ DAG_ID,
  schedule_interval='@once', # Override to match your needs,
  start_date=datetime(2021, 1, 1),
  catchup=False,
- tags=["example"],
+ tags=["example", "sheets"],
 ) as dag:
+ create_bucket = GCSCreateBucketOperator(
+ task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID
+ )
+
  # [START upload_sheet_to_gcs]
  upload_sheet_to_gcs = GoogleSheetsToGCSOperator(
  task_id="upload_sheet_to_gcs",
- destination_bucket=GCS_BUCKET,
+ destination_bucket=BUCKET_NAME,
  spreadsheet_id=SPREADSHEET_ID,
  )
  # [END upload_sheet_to_gcs]
@@ -65,11 +75,35 @@
  # [START upload_gcs_to_sheet]
  upload_gcs_to_sheet = GCSToGoogleSheetsOperator(
  task_id="upload_gcs_to_sheet",
- bucket_name=GCS_BUCKET,
+ bucket_name=BUCKET_NAME,
  object_name="{{ task_instance.xcom_pull('upload_sheet_to_gcs')[0] }}",
  spreadsheet_id=NEW_SPREADSHEET_ID,
  )
  # [END upload_gcs_to_sheet]
 
- create_spreadsheet >> print_spreadsheet_url
- upload_sheet_to_gcs >> upload_gcs_to_sheet
+ delete_bucket = GCSDeleteBucketOperator(
+ task_id="delete_bucket", bucket_name=BUCKET_NAME, trigger_rule=TriggerRule.ALL_DONE
+ )
+
+ (
+ # TEST SETUP
+ create_bucket
+ # TEST BODY
+ >> create_spreadsheet
+ >> print_spreadsheet_url
+ >> upload_sheet_to_gcs
+ >> upload_gcs_to_sheet
+ # TEST TEARDOWN
+ >> delete_bucket
+ )
+
+ from tests.system.utils.watcher import watcher
+
+ # This test needs watcher in order to properly mark success/failure
+ # when "tearDown" task with trigger rule is part of the DAG
+ list(dag.tasks) >> watcher()
+
+from tests.system.utils import get_test_run # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)