[AIRFLOW-5013] Add GCP Data Catalog Hook and operators (#7664)

apache · Mar 9, 2020 · bf9b6b6 · bf9b6b6
1 parent 29e848d
commit bf9b6b6
Show file tree

Hide file tree

Showing 9 changed files with 6,688 additions and 0 deletions.
diff --git a/airflow/providers/google/cloud/example_dags/example_datacatalog.py b/airflow/providers/google/cloud/example_dags/example_datacatalog.py
diff --git a/airflow/providers/google/cloud/hooks/datacatalog.py b/airflow/providers/google/cloud/hooks/datacatalog.py
diff --git a/airflow/providers/google/cloud/operators/datacatalog.py b/airflow/providers/google/cloud/operators/datacatalog.py
diff --git a/docs/conf.py b/docs/conf.py
@@ -280,6 +280,7 @@
  'google-cloud-bigtable': ('https://googleapis.dev/python/bigtable/latest', None),
  'google-cloud-container': ('https://googleapis.dev/python/container/latest', None),
  'google-cloud-core': ('https://googleapis.dev/python/google-cloud-core/latest', None),
+ 'google-cloud-datacatalog': ('https://googleapis.dev/python/datacatalog/latest', None),
  'google-cloud-datastore': ('https://googleapis.dev/python/datastore/latest', None),
  'google-cloud-dlp': ('https://googleapis.dev/python/dlp/latest', None),
  'google-cloud-kms': ('https://googleapis.dev/python/cloudkms/latest', None),

diff --git a/docs/howto/operator/gcp/datacatalog.rst b/docs/howto/operator/gcp/datacatalog.rst
diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst
@@ -602,6 +602,12 @@ These integrations allow you to perform various operations within the Google Clo
  - :mod:`airflow.providers.google.cloud.operators.datafusion`
  -
 
+ * - `Datacatalog <https://cloud.google.com/data-catalog>`__
+ -
+ - :mod:`airflow.providers.google.cloud.hooks.datacatalog`
+ - :mod:`airflow.providers.google.cloud.operators.datacatalog`
+ -
+
  * - `Dataflow <https://cloud.google.com/dataflow/>`__
  -
  - :mod:`airflow.providers.google.cloud.hooks.dataflow`

diff --git a/setup.py b/setup.py
@@ -227,6 +227,7 @@ def write_version(filename: str = os.path.join(*[dirname(__file__), "airflow", "
  'google-cloud-bigquery-datatransfer>=0.4.0',
  'google-cloud-bigtable>=1.0.0',
  'google-cloud-container>=0.1.1',
+ 'google-cloud-datacatalog>=0.5.0',
  'google-cloud-dataproc>=0.5.0',
  'google-cloud-dlp>=0.11.0',
  'google-cloud-kms>=1.2.1',