Replace google_cloud_storage_conn_id by gcp_conn_id when using `G…

…CSHook` (#13851) google_cloud_storage_conn_id parameter has been deprecated by GCSHook, and should be replaced by gcp_conn_id parameter. google_cloud_storage_conn_id was still in use in many Operators. GCSHook renders a DeprecationWarning message everytime one of those operators uses google_cloud_storage_conn_id. This PR avoid triggering DeprecationWarning when using GCSHook in the codebase. Co-authored-by: Kamil Breguła <[email protected]>
apache · Jan 24, 2021 · f473ca7 · f473ca7
1 parent 910ba25
commit f473ca7
Show file tree

Hide file tree

Showing 26 changed files with 40 additions and 42 deletions.
diff --git a/airflow/providers/amazon/aws/transfers/gcs_to_s3.py b/airflow/providers/amazon/aws/transfers/gcs_to_s3.py
@@ -140,7 +140,7 @@ def __init__(
  def execute(self, context) -> List[str]:
  # list all files in an Google Cloud Storage bucket
  hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.google_impersonation_chain,
  )

diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py
@@ -933,7 +933,7 @@ def execute(self, context) -> None:
  if not self.schema_fields and self.gcs_schema_object:
  gcs_bucket, gcs_object = _parse_gcs_url(self.gcs_schema_object)
  gcs_hook = GCSHook(
- google_cloud_storage_conn_id=self.google_cloud_storage_conn_id,
+ gcp_conn_id=self.google_cloud_storage_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )
@@ -1174,7 +1174,7 @@ def execute(self, context) -> None:
 
  if not self.schema_fields and self.schema_object and self.source_format != 'DATASTORE_BACKUP':
  gcs_hook = GCSHook(
- google_cloud_storage_conn_id=self.google_cloud_storage_conn_id,
+ gcp_conn_id=self.google_cloud_storage_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )

diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py
@@ -1464,9 +1464,7 @@ def _upload_file_temp(self, bucket, local_file):
 
  self.log.info("Uploading %s to %s", local_file, temp_filename)
 
- GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain
- ).upload(
+ GCSHook(gcp_conn_id=self.gcp_conn_id, impersonation_chain=self.impersonation_chain).upload(
  bucket_name=bucket,
  object_name=temp_filename,
  mime_type='application/x-python',

diff --git a/airflow/providers/google/cloud/operators/gcs.py b/airflow/providers/google/cloud/operators/gcs.py
@@ -151,7 +151,7 @@ def __init__(
 
  def execute(self, context) -> None:
  hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )
@@ -259,7 +259,7 @@ def __init__(
  def execute(self, context) -> list:
 
  hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )
@@ -351,7 +351,7 @@ def __init__(
 
  def execute(self, context):
  hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )
@@ -445,7 +445,7 @@ def __init__(
 
  def execute(self, context) -> None:
  hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  impersonation_chain=self.impersonation_chain,
  )
  hook.insert_bucket_acl(
@@ -544,7 +544,7 @@ def __init__(
 
  def execute(self, context) -> None:
  hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  impersonation_chain=self.impersonation_chain,
  )
  hook.insert_object_acl(
@@ -805,7 +805,7 @@ def __init__(
 
  def execute(self, context) -> None:
  hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )

diff --git a/airflow/providers/google/cloud/operators/text_to_speech.py b/airflow/providers/google/cloud/operators/text_to_speech.py
@@ -143,7 +143,7 @@ def execute(self, context) -> None:
  with NamedTemporaryFile() as temp_file:
  temp_file.write(result.audio_content)
  cloud_storage_hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  impersonation_chain=self.impersonation_chain,
  )
  cloud_storage_hook.upload(

diff --git a/airflow/providers/google/cloud/sensors/gcs.py b/airflow/providers/google/cloud/sensors/gcs.py
@@ -83,7 +83,7 @@ def __init__(
  def poke(self, context: dict) -> bool:
  self.log.info('Sensor checks existence of : %s, %s', self.bucket, self.object)
  hook = GCSHook(
- google_cloud_storage_conn_id=self.google_cloud_conn_id,
+ gcp_conn_id=self.google_cloud_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )
@@ -160,7 +160,7 @@ def __init__(
  def poke(self, context: dict) -> bool:
  self.log.info('Sensor checks existence of : %s, %s', self.bucket, self.object)
  hook = GCSHook(
- google_cloud_storage_conn_id=self.google_cloud_conn_id,
+ gcp_conn_id=self.google_cloud_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )
@@ -226,7 +226,7 @@ def __init__(
  def poke(self, context: dict) -> bool:
  self.log.info('Sensor checks existence of objects: %s, %s', self.bucket, self.prefix)
  hook = GCSHook(
- google_cloud_storage_conn_id=self.google_cloud_conn_id,
+ gcp_conn_id=self.google_cloud_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )

diff --git a/airflow/providers/google/cloud/transfers/adls_to_gcs.py b/airflow/providers/google/cloud/transfers/adls_to_gcs.py
@@ -150,7 +150,7 @@ def execute(self, context):
  # use the super to list all files in an Azure Data Lake path
  files = super().execute(context)
  g_hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.google_impersonation_chain,
  )

diff --git a/airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py b/airflow/providers/google/cloud/transfers/azure_fileshare_to_gcs.py
@@ -125,7 +125,7 @@ def execute(self, context):
  )
 
  gcs_hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.google_impersonation_chain,
  )

diff --git a/airflow/providers/google/cloud/transfers/cassandra_to_gcs.py b/airflow/providers/google/cloud/transfers/cassandra_to_gcs.py
@@ -232,7 +232,7 @@ def _write_local_schema_file(self, cursor):
 
  def _upload_to_gcs(self, files_to_upload: Dict[str, Any]):
  hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )

diff --git a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py
@@ -260,7 +260,7 @@ def execute(self, context):
  if not self.schema_fields:
  if self.schema_object and self.source_format != 'DATASTORE_BACKUP':
  gcs_hook = GCSHook(
- google_cloud_storage_conn_id=self.google_cloud_storage_conn_id,
+ gcp_conn_id=self.google_cloud_storage_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )

diff --git a/airflow/providers/google/cloud/transfers/gcs_to_gcs.py b/airflow/providers/google/cloud/transfers/gcs_to_gcs.py
@@ -234,7 +234,7 @@ def __init__(
  def execute(self, context):
 
  hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )

diff --git a/airflow/providers/google/cloud/transfers/gcs_to_local.py b/airflow/providers/google/cloud/transfers/gcs_to_local.py
@@ -129,7 +129,7 @@ def __init__(
  def execute(self, context):
  self.log.info('Executing download: %s, %s, %s', self.bucket, self.object, self.filename)
  hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )

diff --git a/airflow/providers/google/cloud/transfers/local_to_gcs.py b/airflow/providers/google/cloud/transfers/local_to_gcs.py
@@ -111,7 +111,7 @@ def __init__(
  def execute(self, context):
  """Uploads a file or list of files to Google Cloud Storage"""
  hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )

diff --git a/airflow/providers/google/cloud/transfers/s3_to_gcs.py b/airflow/providers/google/cloud/transfers/s3_to_gcs.py
@@ -164,7 +164,7 @@ def execute(self, context):
  files = super().execute(context)
 
  gcs_hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.google_impersonation_chain,
  )

diff --git a/airflow/providers/google/marketing_platform/operators/campaign_manager.py b/airflow/providers/google/marketing_platform/operators/campaign_manager.py
@@ -235,7 +235,7 @@ def execute(self, context: dict) -> None:
  impersonation_chain=self.impersonation_chain,
  )
  gcs_hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )

diff --git a/airflow/providers/google/marketing_platform/operators/display_video.py b/airflow/providers/google/marketing_platform/operators/display_video.py
@@ -289,7 +289,7 @@ def execute(self, context: dict):
  impersonation_chain=self.impersonation_chain,
  )
  gcs_hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )

diff --git a/airflow/providers/google/suite/transfers/gcs_to_gdrive.py b/airflow/providers/google/suite/transfers/gcs_to_gdrive.py
@@ -117,7 +117,7 @@ def __init__(
  def execute(self, context):
 
  self.gcs_hook = GCSHook(
- google_cloud_storage_conn_id=self.gcp_conn_id,
+ gcp_conn_id=self.gcp_conn_id,
  delegate_to=self.delegate_to,
  impersonation_chain=self.impersonation_chain,
  )

diff --git a/tests/providers/google/cloud/operators/test_gcs.py b/tests/providers/google/cloud/operators/test_gcs.py
@@ -235,7 +235,7 @@ def test_execute(self, mock_hook):
  )
  task.execute({})
  mock_hook.assert_called_once_with(
- google_cloud_storage_conn_id='GCP_CONN_ID',
+ gcp_conn_id='GCP_CONN_ID',
  delegate_to='DELEGATE_TO',
  impersonation_chain=IMPERSONATION_CHAIN,
  )

diff --git a/tests/providers/google/cloud/operators/test_text_to_speech.py b/tests/providers/google/cloud/operators/test_text_to_speech.py
@@ -62,7 +62,7 @@ def test_synthesize_text_green_path(self, mock_text_to_speech_hook, mock_gcp_hoo
  impersonation_chain=IMPERSONATION_CHAIN,
  )
  mock_gcp_hook.assert_called_once_with(
- google_cloud_storage_conn_id="gcp-conn-id",
+ gcp_conn_id="gcp-conn-id",
  impersonation_chain=IMPERSONATION_CHAIN,
  )
  mock_text_to_speech_hook.return_value.synthesize_speech.assert_called_once_with(

diff --git a/tests/providers/google/cloud/sensors/test_gcs.py b/tests/providers/google/cloud/sensors/test_gcs.py
@@ -79,7 +79,7 @@ def test_should_pass_argument_to_hook(self, mock_hook):
  assert result is True
  mock_hook.assert_called_once_with(
  delegate_to=TEST_DELEGATE_TO,
- google_cloud_storage_conn_id=TEST_GCP_CONN_ID,
+ gcp_conn_id=TEST_GCP_CONN_ID,
  impersonation_chain=TEST_IMPERSONATION_CHAIN,
  )
  mock_hook.return_value.exists.assert_called_once_with(TEST_BUCKET, TEST_OBJECT)
@@ -121,7 +121,7 @@ def test_should_pass_argument_to_hook(self, mock_hook):
 
  mock_hook.assert_called_once_with(
  delegate_to=TEST_DELEGATE_TO,
- google_cloud_storage_conn_id=TEST_GCP_CONN_ID,
+ gcp_conn_id=TEST_GCP_CONN_ID,
  impersonation_chain=TEST_IMPERSONATION_CHAIN,
  )
  mock_hook.return_value.is_updated_after.assert_called_once_with(TEST_BUCKET, TEST_OBJECT, mock.ANY)
@@ -144,7 +144,7 @@ def test_should_pass_arguments_to_hook(self, mock_hook):
 
  mock_hook.assert_called_once_with(
  delegate_to=TEST_DELEGATE_TO,
- google_cloud_storage_conn_id=TEST_GCP_CONN_ID,
+ gcp_conn_id=TEST_GCP_CONN_ID,
  impersonation_chain=TEST_IMPERSONATION_CHAIN,
  )
  mock_hook.return_value.list.assert_called_once_with(TEST_BUCKET, prefix=TEST_PREFIX)
@@ -182,7 +182,7 @@ def test_execute(self, mock_hook):
 
  mock_hook.assert_called_once_with(
  delegate_to=TEST_DELEGATE_TO,
- google_cloud_storage_conn_id=TEST_GCP_CONN_ID,
+ gcp_conn_id=TEST_GCP_CONN_ID,
  impersonation_chain=TEST_IMPERSONATION_CHAIN,
  )
  mock_hook.return_value.list.assert_called_once_with(TEST_BUCKET, prefix=TEST_PREFIX)

diff --git a/tests/providers/google/cloud/transfers/test_adls_to_gcs.py b/tests/providers/google/cloud/transfers/test_adls_to_gcs.py
@@ -95,7 +95,7 @@ def test_execute(self, gcs_mock_hook, adls_one_mock_hook, adls_two_mock_hook):
  adls_one_mock_hook.assert_called_once_with(azure_data_lake_conn_id=AZURE_CONN_ID)
  adls_two_mock_hook.assert_called_once_with(azure_data_lake_conn_id=AZURE_CONN_ID)
  gcs_mock_hook.assert_called_once_with(
- google_cloud_storage_conn_id=GCS_CONN_ID,
+ gcp_conn_id=GCS_CONN_ID,
  delegate_to=None,
  impersonation_chain=IMPERSONATION_CHAIN,
  )

diff --git a/tests/providers/google/cloud/transfers/test_azure_fileshare_to_gcs.py b/tests/providers/google/cloud/transfers/test_azure_fileshare_to_gcs.py
@@ -83,7 +83,7 @@ def test_execute(self, gcs_mock_hook, azure_fileshare_mock_hook):
  azure_fileshare_mock_hook.assert_called_once_with(WASB_CONN_ID)
 
  gcs_mock_hook.assert_called_once_with(
- google_cloud_storage_conn_id=GCS_CONN_ID,
+ gcp_conn_id=GCS_CONN_ID,
  delegate_to=None,
  impersonation_chain=IMPERSONATION_CHAIN,
  )

diff --git a/tests/providers/google/cloud/transfers/test_s3_to_gcs.py b/tests/providers/google/cloud/transfers/test_s3_to_gcs.py
@@ -86,7 +86,7 @@ def test_execute(self, gcs_mock_hook, s3_one_mock_hook, s3_two_mock_hook):
  s3_one_mock_hook.assert_called_once_with(aws_conn_id=AWS_CONN_ID, verify=None)
  s3_two_mock_hook.assert_called_once_with(aws_conn_id=AWS_CONN_ID, verify=None)
  gcs_mock_hook.assert_called_once_with(
- google_cloud_storage_conn_id=GCS_CONN_ID,
+ gcp_conn_id=GCS_CONN_ID,
  delegate_to=None,
  impersonation_chain=IMPERSONATION_CHAIN,
  )
@@ -115,7 +115,7 @@ def test_execute_with_gzip(self, gcs_mock_hook, s3_one_mock_hook, s3_two_mock_ho
 
  operator.execute(None)
  gcs_mock_hook.assert_called_once_with(
- google_cloud_storage_conn_id=GCS_CONN_ID,
+ gcp_conn_id=GCS_CONN_ID,
  delegate_to=None,
  impersonation_chain=None,
  )

diff --git a/tests/providers/google/marketing_platform/operators/test_campaign_manager.py b/tests/providers/google/marketing_platform/operators/test_campaign_manager.py
@@ -126,7 +126,7 @@ def test_execute(
  profile_id=profile_id, report_id=report_id, file_id=file_id
  )
  gcs_hook_mock.assert_called_once_with(
- google_cloud_storage_conn_id=GCP_CONN_ID,
+ gcp_conn_id=GCP_CONN_ID,
  delegate_to=None,
  impersonation_chain=None,
  )

diff --git a/tests/providers/google/marketing_platform/operators/test_display_video.py b/tests/providers/google/marketing_platform/operators/test_display_video.py
@@ -148,7 +148,7 @@ def test_execute(
  mock_hook.return_value.get_query.assert_called_once_with(query_id=report_id)
 
  mock_gcs_hook.assert_called_once_with(
- google_cloud_storage_conn_id=GCP_CONN_ID,
+ gcp_conn_id=GCP_CONN_ID,
  delegate_to=None,
  impersonation_chain=None,
  )

diff --git a/tests/providers/google/suite/transfers/test_gcs_to_gdrive.py b/tests/providers/google/suite/transfers/test_gcs_to_gdrive.py
@@ -48,7 +48,7 @@ def test_should_copy_single_file(self, mock_named_temporary_file, mock_gdrive, m
  [
  mock.call(
  delegate_to=None,
- google_cloud_storage_conn_id="google_cloud_default",
+ gcp_conn_id="google_cloud_default",
  impersonation_chain=None,
  ),
  mock.call().download(
@@ -93,7 +93,7 @@ def test_should_copy_files(self, mock_named_temporary_file, mock_gdrive, mock_gc
  [
  mock.call(
  delegate_to=None,
- google_cloud_storage_conn_id="google_cloud_default",
+ gcp_conn_id="google_cloud_default",
  impersonation_chain=IMPERSONATION_CHAIN,
  ),
  mock.call().list("data", delimiter=".avro", prefix="sales/sales-2017/"),
@@ -137,7 +137,7 @@ def test_should_move_files(self, mock_named_temporary_file, mock_gdrive, mock_gc
  [
  mock.call(
  delegate_to=None,
- google_cloud_storage_conn_id="google_cloud_default",
+ gcp_conn_id="google_cloud_default",
  impersonation_chain=IMPERSONATION_CHAIN,
  ),
  mock.call().list("data", delimiter=".avro", prefix="sales/sales-2017/"),