Fixup docstring for deprecated DataprocSubmitPigJobOperator (#32739)

apache · Jul 21, 2023 · fbeddc3 · fbeddc3
1 parent 440c9eb
commit fbeddc3
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 4 deletions.
diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py
@@ -1096,6 +1096,10 @@ def on_kill(self) -> None:
 class DataprocSubmitPigJobOperator(DataprocJobBaseOperator):
  """Start a Pig query Job on a Cloud DataProc cluster.
 
+ .. seealso::
+ This operator is deprecated, please use
+ :class:`~airflow.providers.google.cloud.operators.dataproc.DataprocSubmitJobOperator`:
+
  The parameters of the operation will be passed to the cluster.
 
  It's a good practice to define dataproc_* parameters in the default_args of the dag

diff --git a/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py b/tests/system/providers/google/cloud/dataproc/example_dataproc_pig.py
@@ -31,8 +31,8 @@
 )
 from airflow.utils.trigger_rule import TriggerRule
 
-ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 DAG_ID = "dataproc_pig"
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT")
 
 CLUSTER_NAME = f"cluster-dataproc-pig-{ENV_ID}"
@@ -41,7 +41,6 @@
 
 
 # Cluster definition
-
 CLUSTER_CONFIG = {
  "master_config": {
  "num_instances": 1,
@@ -72,7 +71,7 @@
  schedule="@once",
  start_date=datetime(2021, 1, 1),
  catchup=False,
- tags=["example", "dataproc"],
+ tags=["example", "dataproc", "pig"],
 ) as dag:
  create_cluster = DataprocCreateClusterOperator(
  task_id="create_cluster",
@@ -94,7 +93,14 @@
  trigger_rule=TriggerRule.ALL_DONE,
  )
 
- create_cluster >> pig_task >> delete_cluster
+ (
+ # TEST SETUP
+ create_cluster
+ # TEST BODY
+ >> pig_task
+ # TEST TEARDOWN
+ >> delete_cluster
+ )
 
  from tests.system.utils.watcher import watcher