Add guide for DataprocInstantiateInlineWorkflowTemplateOperator (#22062)

apache · Mar 8, 2022 · dd0724f · dd0724f
1 parent e113459
commit dd0724f
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 3 deletions.
diff --git a/airflow/providers/google/cloud/example_dags/example_dataproc.py b/airflow/providers/google/cloud/example_dags/example_dataproc.py
@@ -33,6 +33,7 @@
  DataprocDeleteBatchOperator,
  DataprocDeleteClusterOperator,
  DataprocGetBatchOperator,
+ DataprocInstantiateInlineWorkflowTemplateOperator,
  DataprocInstantiateWorkflowTemplateOperator,
  DataprocListBatchesOperator,
  DataprocSubmitJobOperator,
@@ -256,6 +257,12 @@
  )
  # [END how_to_cloud_dataproc_trigger_workflow_template]
 
+ # [START how_to_cloud_dataproc_instantiate_inline_workflow_template]
+ instantiate_inline_workflow_template = DataprocInstantiateInlineWorkflowTemplateOperator(
+ task_id='instantiate_inline_workflow_template', template=WORKFLOW_TEMPLATE, region=REGION
+ )
+ # [END how_to_cloud_dataproc_instantiate_inline_workflow_template]
+
  pig_task = DataprocSubmitJobOperator(
  task_id="pig_task", job=PIG_JOB, region=REGION, project_id=PROJECT_ID
  )

diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py
@@ -1673,8 +1673,11 @@ class DataprocInstantiateInlineWorkflowTemplateOperator(BaseOperator):
  wait until the WorkflowTemplate is finished executing.
 
  .. seealso::
- Please refer to:
- https://cloud.google.com/dataproc/docs/reference/rest/v1beta2/projects.regions.workflowTemplates/instantiateInline
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:DataprocInstantiateInlineWorkflowTemplateOperator`
+
+ For more detail on about instantiate inline have a look at the reference:
+ https://cloud.google.com/dataproc/docs/reference/rest/v1/projects.regions.workflowTemplates/instantiateInline
 
  :param template: The template contents. (templated)
  :param project_id: The ID of the google cloud project in which

diff --git a/docs/apache-airflow-providers-google/operators/cloud/dataproc.rst b/docs/apache-airflow-providers-google/operators/cloud/dataproc.rst
@@ -32,6 +32,7 @@ Prerequisite Tasks
 
 
 .. _howto/operator:DataprocCreateClusterOperator:
+.. _howto/operator:DataprocInstantiateInlineWorkflowTemplateOperator:
 
 Create a Cluster
 ----------------
@@ -212,6 +213,15 @@ Once a workflow is created users can trigger it using
  :start-after: [START how_to_cloud_dataproc_trigger_workflow_template]
  :end-before: [END how_to_cloud_dataproc_trigger_workflow_template]
 
+The inline operator is an alternative. It creates a workflow, run it, and delete it afterwards:
+:class:`~airflow.providers.google.cloud.operators.dataproc.DataprocInstantiateInlineWorkflowTemplateOperator`:
+
+.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dataproc.py
+ :language: python
+ :dedent: 4
+ :start-after: [START how_to_cloud_dataproc_instantiate_inline_workflow_template]
+ :end-before: [END how_to_cloud_dataproc_instantiate_inline_workflow_template]
+
 Create a Batch
 --------------
 

diff --git a/tests/always/test_project_structure.py b/tests/always/test_project_structure.py
@@ -192,7 +192,6 @@ class TestGoogleProviderProjectStructure(unittest.TestCase):
 
  # Please at the examples to those operators at the earliest convenience :)
  MISSING_EXAMPLES_FOR_OPERATORS = {
- 'airflow.providers.google.cloud.operators.dataproc.DataprocInstantiateInlineWorkflowTemplateOperator',
  'airflow.providers.google.cloud.operators.mlengine.MLEngineTrainingCancelJobOperator',
  'airflow.providers.google.cloud.operators.dlp.CloudDLPGetStoredInfoTypeOperator',
  'airflow.providers.google.cloud.operators.dlp.CloudDLPReidentifyContentOperator',