Skip to content

Commit

Permalink
D401 lint fixes for all hooks in google provider (#37296)
Browse files Browse the repository at this point in the history 
* google provider - few D401 fixes

Signed-off-by: kalyanr <[email protected]>

* few more google provider D401 fixes

Signed-off-by: kalyanr <[email protected]>

* D401 fixes

Signed-off-by: kalyanr <[email protected]>

* include

Signed-off-by: kalyanr <[email protected]>

* add D401 fixes

Signed-off-by: kalyanr <[email protected]>

* add

Signed-off-by: kalyanr <[email protected]>

* update deprecated message in bigquery hooks

Signed-off-by: kalyanr <[email protected]>

* fix

Signed-off-by: kalyanr <[email protected]>

* fix

Signed-off-by: kalyanr <[email protected]>

* few more D401 fixes

Signed-off-by: kalyanr <[email protected]>

* remove D401 for fixed hooks from pyproject.toml

Signed-off-by: kalyanr <[email protected]>

* fix missed

Signed-off-by: kalyanr <[email protected]>

* fix D401 in few hooks

Signed-off-by: kalyanr <[email protected]>

* fix D401 in rest of the google providers

Signed-off-by: kalyanr <[email protected]>

---------

Signed-off-by: kalyanr <[email protected]>
  • Loading branch information
@rawwar
rawwar committed Feb 11, 2024
1 parent f61ffe5 commit 9c4a9d7
Show file tree
Hide file tree
Showing 52 changed files with 486 additions and 533 deletions.
26 changes: 13 additions & 13 deletions airflow/providers/google/cloud/hooks/automl.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,12 +83,12 @@ def __init__(

@staticmethod
def extract_object_id(obj: dict) -> str:
"""Returns unique id of the object."""
"""Return unique id of the object."""
return obj["name"].rpartition("/")[-1]

def get_conn(self) -> AutoMlClient:
"""
Retrieves connection to AutoML.
Retrieve connection to AutoML.
:return: Google Cloud AutoML client object.
"""
Expand All @@ -97,7 +97,7 @@ def get_conn(self) -> AutoMlClient:
return self._client

def wait_for_operation(self, operation: Operation, timeout: float | None = None):
"""Waits for long-lasting operation to complete."""
"""Wait for long-lasting operation to complete."""
try:
return operation.result(timeout=timeout)
except Exception:
Expand All @@ -124,7 +124,7 @@ def create_model(
retry: Retry | _MethodDefault = DEFAULT,
) -> Operation:
"""
Creates a model_id and returns a Model in the `response` field when it completes.
Create a model_id and returns a Model in the `response` field when it completes.
When you create a model, several model evaluations are created for it:
a global evaluation, and one evaluation for each annotation spec.
Expand Down Expand Up @@ -257,7 +257,7 @@ def create_dataset(
metadata: Sequence[tuple[str, str]] = (),
) -> Dataset:
"""
Creates a dataset.
Create a dataset.
:param dataset: The dataset to create. If a dict is provided, it must be of the
same form as the protobuf message Dataset.
Expand Down Expand Up @@ -294,7 +294,7 @@ def import_data(
metadata: Sequence[tuple[str, str]] = (),
) -> Operation:
"""
Imports data into a dataset. For Tables this method can only be called on an empty Dataset.
Import data into a dataset. For Tables this method can only be called on an empty Dataset.
:param dataset_id: Name of the AutoML dataset.
:param input_config: The desired input location and its domain specific semantics, if any.
Expand Down Expand Up @@ -335,7 +335,7 @@ def list_column_specs(
metadata: Sequence[tuple[str, str]] = (),
) -> ListColumnSpecsPager:
"""
Lists column specs in a table spec.
List column specs in a table spec.
:param dataset_id: Name of the AutoML dataset.
:param table_spec_id: table_spec_id for path builder.
Expand Down Expand Up @@ -384,7 +384,7 @@ def get_model(
metadata: Sequence[tuple[str, str]] = (),
) -> Model:
"""
Gets a AutoML model.
Get a AutoML model.
:param model_id: Name of the model.
:param project_id: ID of the Google Cloud project where model is located if None then
Expand Down Expand Up @@ -419,7 +419,7 @@ def delete_model(
metadata: Sequence[tuple[str, str]] = (),
) -> Operation:
"""
Deletes a AutoML model.
Delete a AutoML model.
:param model_id: Name of the model.
:param project_id: ID of the Google Cloud project where model is located if None then
Expand Down Expand Up @@ -452,7 +452,7 @@ def update_dataset(
metadata: Sequence[tuple[str, str]] = (),
) -> Dataset:
"""
Updates a dataset.
Update a dataset.
:param dataset: The dataset which replaces the resource on the server.
If a dict is provided, it must be of the same form as the protobuf message Dataset.
Expand Down Expand Up @@ -536,7 +536,7 @@ def list_table_specs(
metadata: Sequence[tuple[str, str]] = (),
) -> ListTableSpecsPager:
"""
Lists table specs in a dataset_id.
List table specs in a dataset_id.
:param dataset_id: Name of the dataset.
:param filter_: Filter expression, see go/filtering.
Expand Down Expand Up @@ -579,7 +579,7 @@ def list_datasets(
metadata: Sequence[tuple[str, str]] = (),
) -> ListDatasetsPager:
"""
Lists datasets in a project.
List datasets in a project.
:param project_id: ID of the Google Cloud project where dataset is located if None then
default project_id is used.
Expand Down Expand Up @@ -616,7 +616,7 @@ def delete_dataset(
metadata: Sequence[tuple[str, str]] = (),
) -> Operation:
"""
Deletes a dataset and all of its contents.
Delete a dataset and all of its contents.
:param dataset_id: ID of dataset to be deleted.
:param project_id: ID of the Google Cloud project where dataset is located if None then
Expand Down
66 changes: 33 additions & 33 deletions airflow/providers/google/cloud/hooks/bigquery.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2348,17 +2348,17 @@ def __init__(self, *args, **kwargs) -> None:
self._kwargs = kwargs

def close(self) -> None:
"""The BigQueryConnection does not have anything to close."""
"""Do nothing. Not needed for BigQueryConnection."""

def commit(self) -> None:
"""The BigQueryConnection does not support transactions."""
"""Do nothing. BigQueryConnection does not support transactions."""

def cursor(self) -> BigQueryCursor:
"""Return a new :py:class:`Cursor` object using the connection."""
return BigQueryCursor(*self._args, **self._kwargs)

def rollback(self) -> NoReturn:
"""The BigQueryConnection does not have transactions."""
"""Do nothing. BigQueryConnection does not support transactions."""
raise NotImplementedError("BigQueryConnection does not have transactions")


Expand Down Expand Up @@ -2399,7 +2399,7 @@ def __init__(
category=AirflowProviderDeprecationWarning,
)
def create_empty_table(self, *args, **kwargs):
"""This method is deprecated.
"""Create empty table. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.create_empty_table`
instead.
Expand All @@ -2411,7 +2411,7 @@ def create_empty_table(self, *args, **kwargs):
category=AirflowProviderDeprecationWarning,
)
def create_empty_dataset(self, *args, **kwargs) -> dict[str, Any]:
"""This method is deprecated.
"""Create empty dataset. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.create_empty_dataset`
instead.
Expand All @@ -2423,7 +2423,7 @@ def create_empty_dataset(self, *args, **kwargs) -> dict[str, Any]:
category=AirflowProviderDeprecationWarning,
)
def get_dataset_tables(self, *args, **kwargs) -> list[dict[str, Any]]:
"""This method is deprecated.
"""Get dataset tables. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.get_dataset_tables`
instead.
Expand All @@ -2435,7 +2435,7 @@ def get_dataset_tables(self, *args, **kwargs) -> list[dict[str, Any]]:
category=AirflowProviderDeprecationWarning,
)
def delete_dataset(self, *args, **kwargs) -> None:
"""This method is deprecated.
"""Delete dataset. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.delete_dataset`
instead.
Expand All @@ -2447,7 +2447,7 @@ def delete_dataset(self, *args, **kwargs) -> None:
category=AirflowProviderDeprecationWarning,
)
def create_external_table(self, *args, **kwargs):
"""This method is deprecated.
"""Create external table. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.create_external_table`
instead.
Expand All @@ -2459,7 +2459,7 @@ def create_external_table(self, *args, **kwargs):
category=AirflowProviderDeprecationWarning,
)
def patch_table(self, *args, **kwargs) -> None:
"""This method is deprecated.
"""Patch table. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.patch_table`
instead.
Expand All @@ -2471,7 +2471,7 @@ def patch_table(self, *args, **kwargs) -> None:
category=AirflowProviderDeprecationWarning,
)
def insert_all(self, *args, **kwargs) -> None:
"""This method is deprecated.
"""Insert all. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.insert_all`
instead.
Expand All @@ -2483,7 +2483,7 @@ def insert_all(self, *args, **kwargs) -> None:
category=AirflowProviderDeprecationWarning,
)
def update_dataset(self, *args, **kwargs) -> dict:
"""This method is deprecated.
"""Update dataset. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.update_dataset`
instead.
Expand All @@ -2495,7 +2495,7 @@ def update_dataset(self, *args, **kwargs) -> dict:
category=AirflowProviderDeprecationWarning,
)
def patch_dataset(self, *args, **kwargs) -> dict:
"""This method is deprecated.
"""Patch dataset. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.patch_dataset`
instead.
Expand All @@ -2507,7 +2507,7 @@ def patch_dataset(self, *args, **kwargs) -> dict:
category=AirflowProviderDeprecationWarning,
)
def get_dataset_tables_list(self, *args, **kwargs) -> list[dict[str, Any]]:
"""This method is deprecated.
"""Get dataset tables list. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.get_dataset_tables_list`
instead.
Expand All @@ -2519,7 +2519,7 @@ def get_dataset_tables_list(self, *args, **kwargs) -> list[dict[str, Any]]:
category=AirflowProviderDeprecationWarning,
)
def get_datasets_list(self, *args, **kwargs) -> list | HTTPIterator:
"""This method is deprecated.
"""Get datasets list. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.get_datasets_list`
instead.
Expand All @@ -2531,7 +2531,7 @@ def get_datasets_list(self, *args, **kwargs) -> list | HTTPIterator:
category=AirflowProviderDeprecationWarning,
)
def get_dataset(self, *args, **kwargs) -> Dataset:
"""This method is deprecated.
"""Get dataset. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.get_dataset`
instead.
Expand All @@ -2543,7 +2543,7 @@ def get_dataset(self, *args, **kwargs) -> Dataset:
category=AirflowProviderDeprecationWarning,
)
def run_grant_dataset_view_access(self, *args, **kwargs) -> dict:
"""This method is deprecated.
"""Grant view access to dataset. DEPRECATED.
Please use
:func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.run_grant_dataset_view_access`
Expand All @@ -2556,7 +2556,7 @@ def run_grant_dataset_view_access(self, *args, **kwargs) -> dict:
category=AirflowProviderDeprecationWarning,
)
def run_table_upsert(self, *args, **kwargs) -> dict:
"""This method is deprecated.
"""Upsert table. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.run_table_upsert`
instead.
Expand All @@ -2568,7 +2568,7 @@ def run_table_upsert(self, *args, **kwargs) -> dict:
category=AirflowProviderDeprecationWarning,
)
def run_table_delete(self, *args, **kwargs) -> None:
"""This method is deprecated.
"""Delete table. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.run_table_delete`
instead.
Expand All @@ -2580,7 +2580,7 @@ def run_table_delete(self, *args, **kwargs) -> None:
category=AirflowProviderDeprecationWarning,
)
def get_tabledata(self, *args, **kwargs) -> list[dict]:
"""This method is deprecated.
"""Get table data. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.get_tabledata`
instead.
Expand All @@ -2592,7 +2592,7 @@ def get_tabledata(self, *args, **kwargs) -> list[dict]:
category=AirflowProviderDeprecationWarning,
)
def get_schema(self, *args, **kwargs) -> dict:
"""This method is deprecated.
"""Get Schema. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.get_schema`
instead.
Expand All @@ -2604,7 +2604,7 @@ def get_schema(self, *args, **kwargs) -> dict:
category=AirflowProviderDeprecationWarning,
)
def poll_job_complete(self, *args, **kwargs) -> bool:
"""This method is deprecated.
"""Poll for job completion.DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.poll_job_complete`
instead.
Expand All @@ -2616,7 +2616,7 @@ def poll_job_complete(self, *args, **kwargs) -> bool:
category=AirflowProviderDeprecationWarning,
)
def cancel_query(self, *args, **kwargs) -> None:
"""This method is deprecated.
"""Cancel query. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.cancel_query`
instead.
Expand All @@ -2628,7 +2628,7 @@ def cancel_query(self, *args, **kwargs) -> None:
category=AirflowProviderDeprecationWarning,
)
def run_with_configuration(self, *args, **kwargs) -> str:
"""This method is deprecated.
"""Run with configuration. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.run_with_configuration`
instead.
Expand All @@ -2640,7 +2640,7 @@ def run_with_configuration(self, *args, **kwargs) -> str:
category=AirflowProviderDeprecationWarning,
)
def run_load(self, *args, **kwargs) -> str:
"""This method is deprecated.
"""Run load. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.run_load`
instead.
Expand All @@ -2652,7 +2652,7 @@ def run_load(self, *args, **kwargs) -> str:
category=AirflowProviderDeprecationWarning,
)
def run_copy(self, *args, **kwargs) -> str:
"""This method is deprecated.
"""Run copy. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.run_copy`
instead.
Expand All @@ -2664,7 +2664,7 @@ def run_copy(self, *args, **kwargs) -> str:
category=AirflowProviderDeprecationWarning,
)
def run_extract(self, *args, **kwargs) -> str | BigQueryJob:
"""This method is deprecated.
"""Run extraction. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.run_extract`
instead.
Expand All @@ -2676,7 +2676,7 @@ def run_extract(self, *args, **kwargs) -> str | BigQueryJob:
category=AirflowProviderDeprecationWarning,
)
def run_query(self, *args, **kwargs) -> str:
"""This method is deprecated.
"""Run query. DEPRECATED.
Please use :func:`~airflow.providers.google.cloud.hooks.bigquery.BigQueryHook.run_query`
instead.
Expand Down Expand Up @@ -2844,7 +2844,7 @@ def fetchall(self) -> list[list]:
return result

def get_arraysize(self) -> int:
"""Number of rows to fetch at a time.
"""Get number of rows to fetch at a time.
.. seealso:: :func:`.fetchmany()`
"""
Expand All @@ -2860,10 +2860,10 @@ def set_arraysize(self, arraysize: int) -> None:
arraysize = property(get_arraysize, set_arraysize)

def setinputsizes(self, sizes: Any) -> None:
"""Does nothing by default."""
"""Do nothing by default."""

def setoutputsize(self, size: Any, column: Any = None) -> None:
"""Does nothing by default."""
"""Do nothing by default."""

def _get_query_result(self) -> dict:
"""Get job query results; data, schema, job type, etc."""
Expand Down Expand Up @@ -2915,7 +2915,7 @@ def _prepare_query_configuration(
cluster_fields: list[str] | None = None,
encryption_configuration: dict | None = None,
):
"""Helper method that prepare configuration for query."""
"""Prepare configuration for query."""
labels = labels or self.hook.labels
schema_update_options = list(schema_update_options or [])

Expand Down Expand Up @@ -3047,7 +3047,7 @@ def _prepare_query_configuration(


def _bind_parameters(operation: str, parameters: dict) -> str:
"""Helper method that binds parameters to a SQL query."""
"""Bind parameters to a SQL query."""
# inspired by MySQL Python Connector (conversion.py)
string_parameters = {} # type dict[str, str]
for name, value in parameters.items():
Expand All @@ -3061,7 +3061,7 @@ def _bind_parameters(operation: str, parameters: dict) -> str:


def _escape(s: str) -> str:
"""Helper method that escapes parameters to a SQL query."""
"""Escape special characters in a SQL query string."""
e = s
e = e.replace("\\", "\\\\")
e = e.replace("\n", "\\n")
Expand Down

0 comments on commit 9c4a9d7

Please sign in to comment.