Auto-apply apply_default decorator (#15667)

* Auto-apply apply_default decorator * Automatically patch warnings stacklevel inside Operator constructors This makes two changes to make the apply_defaults transparent to warnings: 1. It removes `__call__` from the Metaclass (which added one frame) 2. It "patches" warnings.warn _in the function scope_ to automatically adjust the stacklevel The reason for this is that now that apply_defaults is called "magically" from the Metaclass we're adding extra frames to the callstack. By locally patching warnings like this it means that user code doesn't have to be aware of what we've done. (I wish Python had a native way of handling this case as it crops up all the time. Perhaps I should submit a PEP for it with a better design) * Don't test provider packges against 2.0 anymore Since we have changed the dep to need Airflow 2.1, testing against 2.0 no longer makes any sense. And since 2.1 is not yet out, we have disabled this specific step for now Co-authored-by: Kamil Breguła <[email protected]>
apache · May 7, 2021 · 37681bc · 37681bc
1 parent db0e942
commit 37681bc
Show file tree

Hide file tree

Showing 333 changed files with 348 additions and 1,215 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -553,11 +553,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
  env:
  USE_AIRFLOW_VERSION: "wheel"
  PACKAGE_FORMAT: "wheel"
- - name: "Install and test provider packages and airflow on Airflow 2.0 files"
+ - name: "Install and test provider packages and airflow on Airflow 2.1 files"
  run: ./scripts/ci/provider_packages/ci_install_and_test_provider_packages.sh
  env:
- USE_AIRFLOW_VERSION: "2.0.0"
+ USE_AIRFLOW_VERSION: "2.1.0"
  PACKAGE_FORMAT: "wheel"
+ if: false
 
  prepare-test-provider-packages-sdist:
  timeout-minutes: 40

diff --git a/UPDATING.md b/UPDATING.md
@@ -71,6 +71,10 @@ https://developers.google.com/style/inclusive-documentation
 
 -->
 
+### `@apply_default` decorator isn't longer necessary
+
+This decorator is now automatically added to all operators via the metaclass on BaseOperator
+
 ### Change the configuration options for field masking
 
 We've improved masking for sensitive data in Web UI and logs. As part of it, the following configurations have been changed:

diff --git a/airflow/contrib/operators/adls_to_gcs.py b/airflow/contrib/operators/adls_to_gcs.py
@@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.transfers.adls_to_gcs.ADLSToGCSOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
diff --git a/airflow/contrib/operators/awsbatch_operator.py b/airflow/contrib/operators/awsbatch_operator.py
@@ -53,7 +53,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.amazon.aws.operators.batch.AwsBatchOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 

diff --git a/airflow/contrib/operators/bigquery_operator.py b/airflow/contrib/operators/bigquery_operator.py
@@ -51,6 +51,6 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.bigquery.BigQueryExecuteQueryOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
diff --git a/airflow/contrib/operators/bigquery_table_delete_operator.py b/airflow/contrib/operators/bigquery_table_delete_operator.py
@@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteTableOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
diff --git a/airflow/contrib/operators/bigquery_to_gcs.py b/airflow/contrib/operators/bigquery_to_gcs.py
@@ -39,6 +39,6 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.transfers.bigquery_to_gcs.BigQueryToGCSOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
diff --git a/airflow/contrib/operators/cassandra_to_gcs.py b/airflow/contrib/operators/cassandra_to_gcs.py
@@ -42,6 +42,6 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.transfers.cassandra_to_gcs.CassandraToGCSOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
diff --git a/airflow/contrib/operators/dataflow_operator.py b/airflow/contrib/operators/dataflow_operator.py
@@ -43,7 +43,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.dataflow.DataflowCreateJavaJobOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -60,7 +60,7 @@ def __init__(self, *args, **kwargs):
  Please use
  `airflow.providers.google.cloud.operators.dataflow.DataflowCreatePythonJobOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -77,6 +77,6 @@ def __init__(self, *args, **kwargs):
  Please use
  `airflow.providers.google.cloud.operators.dataflow.DataflowTemplatedJobStartOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
diff --git a/airflow/contrib/operators/dataproc_operator.py b/airflow/contrib/operators/dataproc_operator.py
@@ -52,7 +52,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.dataproc.DataprocCreateClusterOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -68,7 +68,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.dataproc.DataprocDeleteClusterOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -84,7 +84,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.dataproc.DataprocScaleClusterOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -101,7 +101,7 @@ def __init__(self, *args, **kwargs):
  Please use
  `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitHadoopJobOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -118,7 +118,7 @@ def __init__(self, *args, **kwargs):
  Please use
  `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitHiveJobOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -134,7 +134,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.dataproc.DataprocJobBaseOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -150,7 +150,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitPigJobOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -167,7 +167,7 @@ def __init__(self, *args, **kwargs):
  Please use
  `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitPySparkJobOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -184,7 +184,7 @@ def __init__(self, *args, **kwargs):
  Please use
  `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitSparkJobOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -201,7 +201,7 @@ def __init__(self, *args, **kwargs):
  Please use
  `airflow.providers.google.cloud.operators.dataproc.DataprocSubmitSparkSqlJobOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -220,7 +220,7 @@ def __init__(self, *args, **kwargs):
  `airflow.providers.google.cloud.operators.dataproc
  .DataprocInstantiateInlineWorkflowTemplateOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -239,6 +239,6 @@ def __init__(self, *args, **kwargs):
  `airflow.providers.google.cloud.operators.dataproc
  .DataprocInstantiateWorkflowTemplateOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
diff --git a/airflow/contrib/operators/datastore_export_operator.py b/airflow/contrib/operators/datastore_export_operator.py
@@ -40,6 +40,6 @@ def __init__(self, *args, **kwargs):
  Please use
  `airflow.providers.google.cloud.operators.datastore.CloudDatastoreExportEntitiesOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
diff --git a/airflow/contrib/operators/datastore_import_operator.py b/airflow/contrib/operators/datastore_import_operator.py
@@ -40,6 +40,6 @@ def __init__(self, *args, **kwargs):
  Please use
  `airflow.providers.google.cloud.operators.datastore.CloudDatastoreImportEntitiesOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
diff --git a/airflow/contrib/operators/file_to_gcs.py b/airflow/contrib/operators/file_to_gcs.py
@@ -40,6 +40,6 @@ def __init__(self, *args, **kwargs):
  Please use
  `airflow.providers.google.cloud.transfers.local_to_gcs.LocalFilesystemToGCSOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
diff --git a/airflow/contrib/operators/gcp_bigtable_operator.py b/airflow/contrib/operators/gcp_bigtable_operator.py
@@ -50,7 +50,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.bigtable.BigtableUpdateClusterOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -66,7 +66,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.bigtable.BigtableCreateInstanceOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -82,7 +82,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.bigtable.BigtableDeleteInstanceOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -98,7 +98,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.bigtable.BigtableCreateTableOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -114,7 +114,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.bigtable.BigtableDeleteTableOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -132,6 +132,6 @@ def __init__(self, *args, **kwargs):
  Please use
  `airflow.providers.google.cloud.sensors.bigtable.BigtableTableReplicationCompletedSensor`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
diff --git a/airflow/contrib/operators/gcp_compute_operator.py b/airflow/contrib/operators/gcp_compute_operator.py
@@ -46,7 +46,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.compute.ComputeEngineBaseOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -64,7 +64,7 @@ def __init__(self, *args, **kwargs):
  `airflow.providers.google.cloud.operators.compute
  .ComputeEngineInstanceGroupUpdateManagerTemplateOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -82,7 +82,7 @@ def __init__(self, *args, **kwargs):
  Please use `airflow.providers.google.cloud.operators.compute
  .ComputeEngineStartInstanceOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -99,7 +99,7 @@ def __init__(self, *args, **kwargs):
  Please use `airflow.providers.google.cloud.operators.compute
  .ComputeEngineStopInstanceOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -116,7 +116,7 @@ def __init__(self, *args, **kwargs):
  Please use `airflow.providers.google.cloud.operators.compute
  .ComputeEngineCopyInstanceTemplateOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -133,6 +133,6 @@ def __init__(self, *args, **kwargs):
  Please use `airflow.providers.google.cloud.operators.compute
  .ComputeEngineSetMachineTypeOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
diff --git a/airflow/contrib/operators/gcp_container_operator.py b/airflow/contrib/operators/gcp_container_operator.py
@@ -46,7 +46,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.container.GKECreateClusterOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -62,7 +62,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.container.GKEDeleteClusterOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -78,6 +78,6 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.container.GKEStartPodOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
diff --git a/airflow/contrib/operators/gcp_dlp_operator.py b/airflow/contrib/operators/gcp_dlp_operator.py
@@ -71,7 +71,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteDLPJobOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -87,7 +87,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.dlp.CloudDLPGetDLPJobOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -103,7 +103,7 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.dlp.CloudDLPGetDLPJobTriggerOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)
 
@@ -119,6 +119,6 @@ def __init__(self, *args, **kwargs):
  """This class is deprecated.
  Please use `airflow.providers.google.cloud.operators.dlp.CloudDLPListDLPJobsOperator`.""",
  DeprecationWarning,
- stacklevel=3,
+ stacklevel=2,
  )
  super().__init__(*args, **kwargs)