Added example DAG for MSSQL to Google Cloud Storage (GCS) (#19873)

apache · Dec 6, 2021 · 564fe63 · 564fe63
1 parent 8ca3eda
commit 564fe63
Show file tree

Hide file tree

Showing 5 changed files with 103 additions and 1 deletion.
diff --git a/airflow/providers/google/cloud/example_dags/example_mssql_to_gcs.py b/airflow/providers/google/cloud/example_dags/example_mssql_to_gcs.py
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import os
+from datetime import datetime
+
+from airflow import models
+from airflow.providers.google.cloud.transfers.mssql_to_gcs import MSSQLToGCSOperator
+
+GCS_BUCKET = os.environ.get("GCP_GCS_BUCKET", "example-airflow")
+FILENAME = 'test_file'
+
+SQL_QUERY = "USE airflow SELECT * FROM Country;"
+
+with models.DAG(
+ 'example_mssql_to_gcs',
+ schedule_interval='@once',
+ start_date=datetime(2021, 12, 1),
+ catchup=False,
+ tags=['example'],
+) as dag:
+ # [START howto_operator_mssql_to_gcs]
+ upload = MSSQLToGCSOperator(
+ task_id='mssql_to_gcs',
+ mssql_conn_id='airflow_mssql',
+ sql=SQL_QUERY,
+ bucket=GCS_BUCKET,
+ filename=FILENAME,
+ export_format='csv',
+ )
+ # [END howto_operator_mssql_to_gcs]
diff --git a/airflow/providers/google/cloud/transfers/mssql_to_gcs.py b/airflow/providers/google/cloud/transfers/mssql_to_gcs.py
@@ -46,6 +46,11 @@ class MSSQLToGCSOperator(BaseSQLToGCSOperator):
  google_cloud_storage_conn_id='google_cloud_default',
  dag=dag
  )
+
+ .. seealso::
+ For more information on how to use this operator, take a look at the guide:
+ :ref:`howto/operator:MSSQLToGCSOperator`
+
  """
 
  ui_color = '#e0a98c'

diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml
@@ -759,6 +759,10 @@ transfers:
  - source-integration-name: Google Ads
  target-integration-name: Google Cloud Storage (GCS)
  python-module: airflow.providers.google.ads.transfers.ads_to_gcs
+ - source-integration-name: Microsoft SQL Server (MSSQL)
+ target-integration-name: Google Cloud Storage (GCS)
+ python-module: airflow.providers.google.cloud.transfers.mssql_to_gcs
+ how-to-guide: /docs/apache-airflow-providers-google/operators/transfer/mssql_to_gcs.rst
 
 hook-class-names: # deprecated - to be removed after providers add dependency on Airflow 2.2.0+
  - airflow.providers.google.common.hooks.base_google.GoogleBaseHook

diff --git a/docs/apache-airflow-providers-google/operators/transfer/mssql_to_gcs.rst b/docs/apache-airflow-providers-google/operators/transfer/mssql_to_gcs.rst
@@ -0,0 +1,50 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ .. http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+Microsoft SQL Server To Google Cloud Storage Operator
+=====================================================
+The `Google Cloud Storage <https://cloud.google.com/storage/>`__ (GCS) service is
+used to store large data from various applications. This page shows how to copy
+data from Microsoft SQL Server to GCS.
+
+Prerequisite Tasks
+^^^^^^^^^^^^^^^^^^
+
+.. include::/howto/operator/google/_partials/prerequisite_tasks.rst
+
+.. _howto/operator:MSSQLToGCSOperator:
+
+MSSQLToGCSOperator
+~~~~~~~~~~~~~~~~~~
+
+:class:`~airflow.providers.google.cloud.transfers.mssql_to_gcs.MSSQLToGCSOperator` allows you to upload
+data from Microsoft SQL Server database to GCS.
+
+Below is an example of using this operator to upload data to GCS.
+
+.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_mssql_to_gcs.py
+ :language: python
+ :start-after: [START howto_operator_mssql_to_gcs]
+ :end-before: [END howto_operator_mssql_to_gcs]
+
+
+Reference
+---------
+
+For further information, look at:
+* `Microsoft SQL Server Documentation <https://docs.microsoft.com/en-us/sql/?view=sql-server-ver15>`__
+* `Google Cloud Storage Documentation <https://cloud.google.com/storage/>`__
diff --git a/tests/always/test_project_structure.py b/tests/always/test_project_structure.py
@@ -157,7 +157,6 @@ class TestGoogleProviderProjectStructure(unittest.TestCase):
  ('cloud', 'sql_to_gcs'),
  ('cloud', 'bigquery_to_mysql'),
  ('cloud', 'cassandra_to_gcs'),
- ('cloud', 'mssql_to_gcs'),
  ('suite', 'drive'),
  ('ads', 'ads_to_gcs'),
  }