Detect automatically the lack of reference to the guide in the operat…

…or descriptions (#9290) Co-authored-by: ivan.afonichkin <[email protected]> Co-authored-by: Kamil Breguła <[email protected]>
apache · Jun 18, 2020 · 40bf8f2 · 40bf8f2
1 parent e2b2198
commit 40bf8f2
Show file tree

Hide file tree

Showing 19 changed files with 213 additions and 1 deletion.
diff --git a/airflow/providers/amazon/aws/operators/ecs.py b/airflow/providers/amazon/aws/operators/ecs.py
@@ -61,6 +61,10 @@ class ECSOperator(BaseOperator): # pylint: disable=too-many-instance-attributes
  """
  Execute a task on AWS ECS (Elastic Container Service)
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:ECSOperator`
+
  :param task_definition: the task definition name on Elastic Container Service
  :type task_definition: str
  :param cluster: the cluster name on Elastic Container Service

diff --git a/airflow/providers/amazon/aws/transfers/imap_attachment_to_s3.py b/airflow/providers/amazon/aws/transfers/imap_attachment_to_s3.py
@@ -28,6 +28,10 @@ class ImapAttachmentToS3Operator(BaseOperator):
  """
  Transfers a mail attachment from a mail server into s3 bucket.
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:ImapAttachmentToS3Operator`
+
  :param imap_attachment_name: The file name of the mail attachment that you want to transfer.
  :type imap_attachment_name: str
  :param s3_key: The destination file name in the s3 bucket for the attachment.

diff --git a/airflow/providers/amazon/aws/transfers/s3_to_redshift.py b/airflow/providers/amazon/aws/transfers/s3_to_redshift.py
@@ -27,6 +27,10 @@ class S3ToRedshiftOperator(BaseOperator):
  """
  Executes an COPY command to load files from s3 to Redshift
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:S3ToRedshiftOperator`
+
  :param schema: reference to a specific schema in redshift database
  :type schema: str
  :param table: reference to a specific table in redshift database

diff --git a/airflow/providers/apache/spark/operators/spark_jdbc.py b/airflow/providers/apache/spark/operators/spark_jdbc.py
@@ -29,6 +29,10 @@ class SparkJDBCOperator(SparkSubmitOperator):
  SparkSubmitOperator, it assumes that the "spark-submit" binary is available on the
  PATH.
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:SparkJDBCOperator`
+
  :param spark_app_name: Name of the job (default airflow-spark-jdbc)
  :type spark_app_name: str
  :param spark_conn_id: Connection id as configured in Airflow administration

diff --git a/airflow/providers/apache/spark/operators/spark_sql.py b/airflow/providers/apache/spark/operators/spark_sql.py
@@ -25,6 +25,10 @@ class SparkSqlOperator(BaseOperator):
  """
  Execute Spark SQL query
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:SparkSqlOperator`
+
  :param sql: The SQL query to execute. (templated)
  :type sql: str
  :param conf: arbitrary Spark configuration property

diff --git a/airflow/providers/apache/spark/operators/spark_submit.py b/airflow/providers/apache/spark/operators/spark_submit.py
@@ -29,6 +29,10 @@ class SparkSubmitOperator(BaseOperator):
  It requires that the "spark-submit" binary is in the PATH or the spark-home is set
  in the extra on the connection.
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:SparkSubmitOperator`
+
  :param application: The application that submitted as a job, either jar or py file. (templated)
  :type application: str
  :param conf: Arbitrary Spark configuration properties (templated)

diff --git a/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py b/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py
@@ -39,6 +39,10 @@ class KubernetesPodOperator(BaseOperator): # pylint: disable=too-many-instance-
  """
  Execute a task in a Kubernetes Pod
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:KubernetesPodOperator`
+
  .. note::
  If you use `Google Kubernetes Engine <https://cloud.google.com/kubernetes-engine/>`__, use
  :class:`~airflow.providers.google.cloud.operators.kubernetes_engine.GKEStartPodOperator`, which

diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py
@@ -95,6 +95,10 @@ class BigQueryCheckOperator(CheckOperator):
  first row is evaluated using python ``bool`` casting. If any of the
  values return ``False`` the check is failed and errors out.
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:BigQueryCheckOperator`
+
  Note that Python bool casting evals the following as ``False``:
 
  * ``False``
@@ -168,6 +172,10 @@ class BigQueryValueCheckOperator(ValueCheckOperator):
  """
  Performs a simple value check using sql code.
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:BigQueryValueCheckOperator`
+
  :param sql: the sql to be executed
  :type sql: str
  :param use_legacy_sql: Whether to use legacy SQL (true)
@@ -233,6 +241,10 @@ class BigQueryIntervalCheckOperator(IntervalCheckOperator):
  SELECT {metrics_threshold_dict_key} FROM {table}
  WHERE {date_filter_column}=<date>
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:BigQueryIntervalCheckOperator`
+
  :param table: the table name
  :type table: str
  :param days_back: number of days between ds and the ds we want to check
@@ -305,6 +317,10 @@ class BigQueryGetDataOperator(BaseOperator):
 
  **Example Result**: ``[['Tony', '10'], ['Mike', '20'], ['Steve', '15']]``
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:BigQueryGetDataOperator`
+
  .. note::
  If you pass fields to ``selected_fields`` which are in different order than the
  order of columns already in
@@ -650,6 +666,10 @@ class BigQueryCreateEmptyTableOperator(BaseOperator):
  Google Cloud Storage must be a JSON file with the schema fields in it.
  You can also create a table without schema.
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:BigQueryCreateEmptyTableOperator`
+
  :param project_id: The project to create the table into. (templated)
  :type project_id: str
  :param dataset_id: The dataset to create the table into. (templated)
@@ -854,6 +874,10 @@ class BigQueryCreateExternalTableOperator(BaseOperator):
  point the operator to a Google Cloud Storage object name. The object in
  Google Cloud Storage must be a JSON file with the schema fields in it.
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:BigQueryCreateExternalTableOperator`
+
  :param bucket: The bucket to point the external table to. (templated)
  :type bucket: str
  :param source_objects: List of Google Cloud Storage URIs to point
@@ -1076,6 +1100,10 @@ class BigQueryDeleteDatasetOperator(BaseOperator):
  This operator deletes an existing dataset from your Project in Big query.
  https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets/delete
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:BigQueryDeleteDatasetOperator`
+
  :param project_id: The project id of the dataset.
  :type project_id: str
  :param dataset_id: The dataset to be deleted.
@@ -1150,6 +1178,10 @@ class BigQueryCreateEmptyDatasetOperator(BaseOperator):
  This operator is used to create new dataset for your Project in BigQuery.
  https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#resource
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:BigQueryCreateEmptyDatasetOperator`
+
  :param project_id: The name of the project where we want to create the dataset.
  :type project_id: str
  :param dataset_id: The id of dataset. Don't need to provide, if datasetId in dataset_reference.
@@ -1231,6 +1263,10 @@ class BigQueryGetDatasetOperator(BaseOperator):
  """
  This operator is used to return the dataset specified by dataset_id.
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:BigQueryGetDatasetOperator`
+
  :param dataset_id: The id of dataset. Don't need to provide,
  if datasetId in dataset_reference.
  :type dataset_id: str
@@ -1275,6 +1311,10 @@ class BigQueryGetDatasetTablesOperator(BaseOperator):
  """
  This operator retrieves the list of tables in the specified dataset.
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:BigQueryGetDatasetTablesOperator`
+
  :param dataset_id: the dataset ID of the requested dataset.
  :type dataset_id: str
  :param project_id: (Optional) the project of the requested dataset. If None,
@@ -1327,6 +1367,10 @@ class BigQueryPatchDatasetOperator(BaseOperator):
  This operator is used to patch dataset for your Project in BigQuery.
  It only replaces fields that are provided in the submitted dataset resource.
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:BigQueryPatchDatasetOperator`
+
  :param dataset_id: The id of dataset. Don't need to provide,
  if datasetId in dataset_reference.
  :type dataset_id: str
@@ -1388,6 +1432,10 @@ class BigQueryUpdateDatasetOperator(BaseOperator):
  If no ``fields`` are provided then all fields of provided ``dataset_reources``
  will be used.
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:BigQueryUpdateDatasetOperator`
+
  :param dataset_id: The id of dataset. Don't need to provide,
  if datasetId in dataset_reference.
  :type dataset_id: str
@@ -1447,6 +1495,10 @@ class BigQueryDeleteTableOperator(BaseOperator):
  """
  Deletes BigQuery tables
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:BigQueryDeleteTableOperator`
+
  :param deletion_dataset_table: A dotted
  ``(<project>.|<project>:)<dataset>.<table>`` that indicates which table
  will be deleted. (templated)
@@ -1512,6 +1564,10 @@ class BigQueryUpsertTableOperator(BaseOperator):
  """
  Upsert BigQuery table
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:BigQueryUpsertTableOperator`
+
  :param dataset_id: A dotted
  ``(<project>.|<project>:)<dataset>`` that indicates which dataset
  will be updated. (templated)
@@ -1586,6 +1642,11 @@ class BigQueryInsertJobOperator(BaseOperator):
 
  https://cloud.google.com/bigquery/docs/reference/v2/jobs
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:BigQueryInsertJobOperator`
+
+
  :param configuration: The configuration parameter maps directly to BigQuery's
  configuration field in the job object. For more details see
  https://cloud.google.com/bigquery/docs/reference/v2/jobs

diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py
@@ -428,6 +428,10 @@ class DataprocCreateClusterOperator(BaseOperator):
  for a detailed explanation on the different parameters. Most of the configuration
  parameters detailed in the link are available as a parameter to this operator.
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:DataprocCreateClusterOperator`
+
  :param project_id: The ID of the google cloud project in which
  to create the cluster. (templated)
  :type project_id: str

diff --git a/airflow/providers/google/cloud/operators/life_sciences.py b/airflow/providers/google/cloud/operators/life_sciences.py
@@ -29,6 +29,10 @@ class LifeSciencesRunPipelineOperator(BaseOperator):
  """
  Runs a Life Sciences Pipeline
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:LifeSciencesRunPipelineOperator`
+
  :param body: The request body
  :type body: dict
  :param location: The location of the project

diff --git a/airflow/providers/google/cloud/transfers/gcs_to_local.py b/airflow/providers/google/cloud/transfers/gcs_to_local.py
@@ -34,6 +34,10 @@ class GCSToLocalFilesystemOperator(BaseOperator):
  set the ``store_to_xcom_key`` parameter to True push the file content into xcom. When the file size
  exceeds the maximum size for xcom it is recommended to write to a file.
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:GCSToLocalFilesystemOperator`
+
  :param bucket: The Google Cloud Storage bucket where the object is.
  Must not contain 'gs://' prefix. (templated)
  :type bucket: str

diff --git a/airflow/providers/google/cloud/transfers/local_to_gcs.py b/airflow/providers/google/cloud/transfers/local_to_gcs.py
@@ -30,6 +30,10 @@ class LocalFilesystemToGCSOperator(BaseOperator):
  Uploads a file to Google Cloud Storage.
  Optionally can compress the file for upload.
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:LocalFilesystemToGCSOperator`
+
  :param src: Path to the local file. (templated)
  :type src: str
  :param dst: The object name to set when uploading the file. (templated)

diff --git a/airflow/providers/google/marketing_platform/operators/analytics.py b/airflow/providers/google/marketing_platform/operators/analytics.py
@@ -143,7 +143,7 @@ class GoogleAnalyticsRetrieveAdsLinksListOperator(BaseOperator):
 
  .. seealso::
  For more information on how to use this operator, take a look at the guide:
- :ref:`howto/operator:GoogleAnalyticsListAccountsOperator`
+ :ref:`howto/operator:GoogleAnalyticsRetrieveAdsLinksListOperator`
 
  :param account_id: ID of the account which the given web property belongs to.
  :type account_id: str

diff --git a/airflow/providers/google/suite/transfers/gcs_to_gdrive.py b/airflow/providers/google/suite/transfers/gcs_to_gdrive.py
@@ -41,6 +41,10 @@ class GCSToGoogleDriveOperator(BaseOperator):
 
  https://www.googleapis.com/auth/drive
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:GCSToGoogleDriveOperator`
+
  :param source_bucket: The source Google Cloud Storage bucket where the object is. (templated)
  :type source_bucket: str
  :param source_object: The source name of the object to copy in the Google cloud

diff --git a/airflow/providers/http/operators/http.py b/airflow/providers/http/operators/http.py
@@ -27,6 +27,10 @@ class SimpleHttpOperator(BaseOperator):
  """
  Calls an endpoint on an HTTP system to execute an action
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:SimpleHttpOperator`
+
  :param http_conn_id: The connection to run the operator against
  :type http_conn_id: str
  :param endpoint: The relative part of the full url. (templated)

diff --git a/airflow/providers/http/sensors/http.py b/airflow/providers/http/sensors/http.py
@@ -44,6 +44,9 @@ def response_check(response, task_instance):
 
  HttpSensor(task_id='my_http_sensor', ..., response_check=response_check)
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:HttpSensor`
 
  :param http_conn_id: The connection to run the sensor against
  :type http_conn_id: str

diff --git a/airflow/providers/yandex/operators/yandexcloud_dataproc.py b/airflow/providers/yandex/operators/yandexcloud_dataproc.py
@@ -25,6 +25,10 @@
 class DataprocCreateClusterOperator(BaseOperator):
  """Creates Yandex.Cloud Data Proc cluster.
 
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:DataprocCreateClusterOperator`
+
  :param folder_id: ID of the folder in which cluster should be created.
  :type folder_id: Optional[str]
  :param cluster_name: Cluster name. Must be unique inside the folder.