[AIRFLOW-5741] Move Cloud Natural Language to providers (#6421)

apache · Oct 30, 2019 · 470b2a7 · 470b2a7
1 parent bd3aeb2
commit 470b2a7
Show file tree

Hide file tree

Showing 14 changed files with 63 additions and 56 deletions.
diff --git a/UPDATING.md b/UPDATING.md
@@ -221,7 +221,7 @@ The following table shows changes in import paths.
 |airflow.contrib.hooks.gcp_function_hook.GcfHook |airflow.gcp.hooks.functions.CloudFunctionsHook |
 |airflow.contrib.hooks.gcp_kms_hook.GoogleCloudKMSHook |airflow.gcp.hooks.kms.GoogleCloudKMSHook |
 |airflow.contrib.hooks.gcp_mlengine_hook.MLEngineHook |airflow.gcp.hooks.mlengine.MLEngineHook |
-|airflow.contrib.hooks.gcp_natural_language_hook.CloudNaturalLanguageHook |airflow.gcp.hooks.natural_language.CloudNaturalLanguageHook |
+|airflow.contrib.hooks.gcp_natural_language_hook.CloudNaturalLanguageHook |airflow.providers.google.cloud.hooks.natural_language.CloudNaturalLanguageHook |
 |airflow.contrib.hooks.gcp_pubsub_hook.PubSubHook |airflow.gcp.hooks.pubsub.PubSubHook |
 |airflow.contrib.hooks.gcp_spanner_hook.CloudSpannerHook |airflow.gcp.hooks.spanner.SpannerHook |
 |airflow.contrib.hooks.gcp_speech_to_text_hook.GCPSpeechToTextHook |airflow.gcp.hooks.speech_to_text.GCPSpeechToTextHook |
@@ -315,10 +315,10 @@ The following table shows changes in import paths.
 |airflow.contrib.operators.gcp_dlp_operator.CloudDLPUpdateStoredInfoTypeOperator |airflow.gcp.operators.dlp.CloudDLPUpdateStoredInfoTypeOperator |
 |airflow.contrib.operators.gcp_function_operator.GcfFunctionDeleteOperator |airflow.gcp.operators.functions.GcfFunctionDeleteOperator |
 |airflow.contrib.operators.gcp_function_operator.GcfFunctionDeployOperator |airflow.gcp.operators.functions.GcfFunctionDeployOperator |
-|airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeEntitiesOperator |airflow.gcp.operators.natural_language.CloudLanguageAnalyzeEntitiesOperator |
-|airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeEntitySentimentOperator |airflow.gcp.operators.natural_language.CloudLanguageAnalyzeEntitySentimentOperator |
-|airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeSentimentOperator |airflow.gcp.operators.natural_language.CloudLanguageAnalyzeSentimentOperator |
-|airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageClassifyTextOperator |airflow.gcp.operators.natural_language.CloudLanguageClassifyTextOperator |
+|airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeEntitiesOperator |airflow.providers.google.cloud.operators.natural_language.CloudLanguageAnalyzeEntitiesOperator |
+|airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeEntitySentimentOperator |airflow.providers.google.cloud.operators.natural_language.CloudLanguageAnalyzeEntitySentimentOperator |
+|airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeSentimentOperator |airflow.providers.google.cloud.operators.natural_language.CloudLanguageAnalyzeSentimentOperator |
+|airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageClassifyTextOperator |airflow.providers.google.cloud.operators.natural_language.CloudLanguageClassifyTextOperator |
 |airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDatabaseDeleteOperator |airflow.gcp.operators.spanner.CloudSpannerInstanceDatabaseDeleteOperator |
 |airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDatabaseDeployOperator |airflow.gcp.operators.spanner.CloudSpannerInstanceDatabaseDeployOperator |
 |airflow.contrib.operators.gcp_spanner_operator.CloudSpannerInstanceDatabaseQueryOperator |airflow.gcp.operators.spanner.CloudSpannerInstanceDatabaseQueryOperator |

diff --git a/airflow/contrib/hooks/gcp_natural_language_hook.py b/airflow/contrib/hooks/gcp_natural_language_hook.py
@@ -17,15 +17,15 @@
 # specific language governing permissions and limitations
 # under the License.
 """
-This module is deprecated. Please use `airflow.gcp.hooks.natural_language`.
+This module is deprecated. Please use `airflow.providers.google.cloud.hooks.natural_language`.
 """
 
 import warnings
 
 # pylint: disable=unused-import
-from airflow.gcp.hooks.natural_language import CloudNaturalLanguageHook # noqa
+from airflow.providers.google.cloud.hooks.natural_language import CloudNaturalLanguageHook # noqa
 
 warnings.warn(
- "This module is deprecated. Please use `airflow.gcp.hooks.natural_language`",
+ "This module is deprecated. Please use `airflow.providers.google.cloud.hooks.natural_language`",
  DeprecationWarning, stacklevel=2
 )
diff --git a/airflow/contrib/operators/gcp_natural_language_operator.py b/airflow/contrib/operators/gcp_natural_language_operator.py
@@ -17,18 +17,18 @@
 # specific language governing permissions and limitations
 # under the License.
 """
-This module is deprecated. Please use `airflow.gcp.operators.natural_language`.
+This module is deprecated. Please use `airflow.providers.google.cloud.operators.natural_language`.
 """
 
 import warnings
 
 # pylint: disable=unused-import
-from airflow.gcp.operators.natural_language import ( # noqa
+from airflow.providers.google.cloud.operators.natural_language import ( # noqa
  CloudLanguageAnalyzeEntitiesOperator, CloudLanguageAnalyzeEntitySentimentOperator,
  CloudLanguageAnalyzeSentimentOperator, CloudLanguageClassifyTextOperator,
 )
 
 warnings.warn(
- "This module is deprecated. Please use `airflow.gcp.operators.natural_language`",
+ "This module is deprecated. Please use `airflow.providers.google.cloud.operators.natural_language`",
  DeprecationWarning, stacklevel=2
 )
diff --git a/.../example_dags/example_natural_language.py → .../example_dags/example_natural_language.py b/.../example_dags/example_natural_language.py → .../example_dags/example_natural_language.py
@@ -25,15 +25,14 @@
 
 import airflow
 from airflow import models
-from airflow.gcp.operators.natural_language import (
+from airflow.operators.bash_operator import BashOperator
+from airflow.providers.google.cloud.operators.natural_language import (
  CloudLanguageAnalyzeEntitiesOperator, CloudLanguageAnalyzeEntitySentimentOperator,
  CloudLanguageAnalyzeSentimentOperator, CloudLanguageClassifyTextOperator,
 )
-from airflow.operators.bash_operator import BashOperator
 
 # [START howto_operator_gcp_natural_language_document_text]
-TEXT = """
-Airflow is a platform to programmatically author, schedule and monitor workflows.
+TEXT = """Airflow is a platform to programmatically author, schedule and monitor workflows.
 
 Use Airflow to author workflows as Directed Acyclic Graphs (DAGs) of tasks. The Airflow scheduler executes
  your tasks on an array of workers while following the specified dependencies. Rich command line utilities

diff --git a/airflow/gcp/hooks/natural_language.py → ...rs/google/cloud/hooks/natural_language.py b/airflow/gcp/hooks/natural_language.py → ...rs/google/cloud/hooks/natural_language.py
diff --git a/airflow/gcp/operators/natural_language.py → ...oogle/cloud/operators/natural_language.py b/airflow/gcp/operators/natural_language.py → ...oogle/cloud/operators/natural_language.py
@@ -26,8 +26,8 @@
 from google.cloud.language_v1.types import Document
 from google.protobuf.json_format import MessageToDict
 
-from airflow.gcp.hooks.natural_language import CloudNaturalLanguageHook
 from airflow.models import BaseOperator
+from airflow.providers.google.cloud.hooks.natural_language import CloudNaturalLanguageHook
 
 MetaData = Sequence[Tuple[str, str]]
 

diff --git a/docs/autoapi_templates/index.rst b/docs/autoapi_templates/index.rst
@@ -80,6 +80,8 @@ All operators are in the following packages:
 
  airflow/providers/google/marketing_platform/operators/index
 
+ airflow/providers/google/cloud/operators/index
+
  airflow/providers/google/marketing_platform/sensors/index
 
 
@@ -110,6 +112,8 @@ All hooks are in the following packages:
 
  airflow/providers/google/marketing_platform/hooks/index
 
+ airflow/providers/google/cloud/hooks/index
+
 
 Executors
 ---------

diff --git a/docs/conf.py b/docs/conf.py
@@ -217,6 +217,8 @@
  '_api/airflow/providers/google/cloud/example_dags',
  '_api/airflow/providers/google/marketing_platform/index.rst',
  '_api/airflow/providers/google/marketing_platform/example_dags',
+ '_api/airflow/providers/google/cloud/index.rst',
+ '_api/airflow/providers/google/cloud/example_dags',
  'autoapi_templates',
  'howto/operator/gcp/_partials',
 ]

diff --git a/docs/howto/operator/gcp/natural_language.rst b/docs/howto/operator/gcp/natural_language.rst
@@ -48,14 +48,14 @@ representing text.
 
 Here is an example of document with text provided as a string:
 
-.. exampleinclude:: ../../../../airflow/gcp/example_dags/example_natural_language.py
+.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
  :language: python
  :start-after: [START howto_operator_gcp_natural_language_document_text]
  :end-before: [END howto_operator_gcp_natural_language_document_text]
 
 In addition to supplying string, a document can refer to content stored in Google Cloud Storage.
 
-.. exampleinclude:: ../../../../airflow/gcp/example_dags/example_natural_language.py
+.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
  :language: python
  :start-after: [START howto_operator_gcp_natural_language_document_gcs]
  :end-before: [END howto_operator_gcp_natural_language_document_gcs]
@@ -68,20 +68,20 @@ Analyzing Entities
 Entity Analysis inspects the given text for known entities (proper nouns such as
 public figures, landmarks, etc.), and returns information about those entities.
 Entity analysis is performed with the
-:class:`~airflow.gcp.operators.natural_language.CloudLanguageAnalyzeEntitiesOperator` operator.
+:class:`~airflow.providers.google.cloud.operators.natural_language.CloudLanguageAnalyzeEntitiesOperator` operator.
 
-.. exampleinclude:: ../../../../airflow/gcp/example_dags/example_natural_language.py
+.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
  :language: python
  :dedent: 4
  :start-after: [START howto_operator_gcp_natural_language_analyze_entities]
  :end-before: [END howto_operator_gcp_natural_language_analyze_entities]
 
 You can use :ref:`Jinja templating <jinja-templating>` with
-:template-fields:`airflow.gcp.operators.natural_language.CloudLanguageAnalyzeEntitiesOperator`
+:template-fields:`airflow.providers.google.cloud.operators.natural_language.CloudLanguageAnalyzeEntitiesOperator`
 parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom <concepts:xcom>`, which allows it
 to be used by other operators.
 
-.. exampleinclude:: ../../../../airflow/gcp/example_dags/example_natural_language.py
+.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
  :language: python
  :dedent: 4
  :start-after: [START howto_operator_gcp_natural_language_analyze_entities_result]
@@ -95,21 +95,21 @@ Analyzing Entity Sentiment
 Sentiment Analysis inspects the given text and identifies the prevailing
 emotional opinion within the text, especially to determine a writer's attitude
 as positive, negative, or neutral. Sentiment analysis is performed through
-the :class:`~airflow.gcp.operators.natural_language.CloudLanguageAnalyzeEntitySentimentOperator`
+the :class:`~airflow.providers.google.cloud.operators.natural_language.CloudLanguageAnalyzeEntitySentimentOperator`
 operator.
 
-.. exampleinclude:: ../../../../airflow/gcp/example_dags/example_natural_language.py
+.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
  :language: python
  :dedent: 4
  :start-after: [START howto_operator_gcp_natural_language_analyze_entity_sentiment]
  :end-before: [END howto_operator_gcp_natural_language_analyze_entity_sentiment]
 
 You can use :ref:`Jinja templating <jinja-templating>` with
-:template-fields:`airflow.gcp.operators.natural_language.CloudLanguageAnalyzeEntitiesOperator`
+:template-fields:`airflow.providers.google.cloud.operators.natural_language.CloudLanguageAnalyzeEntitiesOperator`
 parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom <concepts:xcom>`, which allows it
 to be used by other operators.
 
-.. exampleinclude:: ../../../../airflow/gcp/example_dags/example_natural_language.py
+.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
  :language: python
  :dedent: 4
  :start-after: [START howto_operator_gcp_natural_language_analyze_entity_sentiment_result]
@@ -124,21 +124,21 @@ Sentiment Analysis inspects the given text and identifies the prevailing
 emotional opinion within the text, especially to determine a writer's
 attitude as positive, negative, or neutral. Sentiment analysis is performed
 through the
-:class:`~airflow.gcp.operators.natural_language.CloudLanguageAnalyzeSentimentOperator`
+:class:`~airflow.providers.google.cloud.operators.natural_language.CloudLanguageAnalyzeSentimentOperator`
 operator.
 
-.. exampleinclude:: ../../../../airflow/gcp/example_dags/example_natural_language.py
+.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
  :language: python
  :dedent: 4
  :start-after: [START howto_operator_gcp_natural_language_analyze_sentiment]
  :end-before: [END howto_operator_gcp_natural_language_analyze_sentiment]
 
 You can use :ref:`Jinja templating <jinja-templating>` with
-:template-fields:`airflow.gcp.operators.natural_language.CloudLanguageAnalyzeSentimentOperator`
+:template-fields:`airflow.providers.google.cloud.operators.natural_language.CloudLanguageAnalyzeSentimentOperator`
 parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom <concepts:xcom>`, which allows it
 to be used by other operators.
 
-.. exampleinclude:: ../../../../airflow/gcp/example_dags/example_natural_language.py
+.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
  :language: python
  :dedent: 4
  :start-after: [START howto_operator_gcp_natural_language_analyze_sentiment_result]
@@ -152,21 +152,21 @@ Classifying Content
 Content Classification analyzes a document and returns a list of content
 categories that apply to the text found in the document. To classify the
 content in a document, use the
-:class:`~airflow.gcp.operators.natural_language.CloudLanguageClassifyTextOperator`
+:class:`~airflow.providers.google.cloud.operators.natural_language.CloudLanguageClassifyTextOperator`
 operator.
 
-.. exampleinclude:: ../../../../airflow/gcp/example_dags/example_natural_language.py
+.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
  :language: python
  :dedent: 4
  :start-after: [START howto_operator_gcp_natural_language_analyze_classify_text]
  :end-before: [END howto_operator_gcp_natural_language_analyze_classify_text]
 
 You can use :ref:`Jinja templating <jinja-templating>` with
-:template-fields:`airflow.gcp.operators.natural_language.CloudLanguageClassifyTextOperator`
+:template-fields:`airflow.providers.google.cloud.operators.natural_language.CloudLanguageClassifyTextOperator`
 parameters which allows you to dynamically determine values. The result is saved to :ref:`XCom <concepts:xcom>`, which allows it
 to be used by other operators.
 
-.. exampleinclude:: ../../../../airflow/gcp/example_dags/example_natural_language.py
+.. exampleinclude:: ../../../../airflow/providers/google/cloud/example_dags/example_natural_language.py
  :language: python
  :dedent: 4
  :start-after: [START howto_operator_gcp_natural_language_analyze_classify_text_result]

diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst
@@ -601,8 +601,8 @@ These integrations allow you to perform various operations within the Google Clo
 
  * - `Natural Language <https://cloud.google.com/natural-language/>`__
  - :doc:`How to use <howto/operator/gcp/natural_language>`
- - :mod:`airflow.gcp.hooks.natural_language`
- - :mod:`airflow.gcp.operators.natural_language`
+ - :mod:`airflow.providers.google.cloud.hooks.natural_language`
+ - :mod:`airflow.providers.google.cloud.operators.natural_language`
  -
 
  * - `Cloud Pub/Sub <https://cloud.google.com/pubsub/>`__