Add autodetect arg in BQCreateExternalTable Operator (#22710)

* Add autodetect parameter * Update docstring * Update google provider documentation
apache · Apr 4, 2022 · f9e1847 · f9e1847
1 parent 215993b
commit f9e1847
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 0 deletions.
diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py
@@ -964,6 +964,9 @@ class BigQueryCreateExternalTableOperator(BaseOperator):
  :param schema_object: If set, a GCS object path pointing to a .json file that
  contains the schema for the table. (templated)
  :param source_format: File format of the data.
+ :param autodetect: Try to detect schema and format options automatically.
+ The schema_fields and schema_object options will be honored when specified explicitly.
+ https://cloud.google.com/bigquery/docs/schema-detect#schema_auto-detection_for_external_data_sources
  :param compression: [Optional] The compression type of the data source.
  Possible values include GZIP and NONE.
  The default value is NONE.
@@ -1028,6 +1031,7 @@ def __init__(
  schema_fields: Optional[List] = None,
  schema_object: Optional[str] = None,
  source_format: Optional[str] = None,
+ autodetect: bool = False,
  compression: Optional[str] = None,
  skip_leading_rows: Optional[int] = None,
  field_delimiter: Optional[str] = None,
@@ -1057,6 +1061,7 @@ def __init__(
  skip_leading_rows,
  field_delimiter,
  max_bad_records,
+ autodetect,
  quote_character,
  allow_quoted_newlines,
  allow_jagged_rows,
@@ -1116,6 +1121,7 @@ def __init__(
  self.bigquery_conn_id = bigquery_conn_id
  self.google_cloud_storage_conn_id = google_cloud_storage_conn_id
  self.delegate_to = delegate_to
+ self.autodetect = autodetect
 
  self.src_fmt_configs = src_fmt_configs or {}
  self.labels = labels
@@ -1153,6 +1159,7 @@ def execute(self, context: 'Context') -> None:
  schema_fields=schema_fields,
  source_uris=source_uris,
  source_format=self.source_format,
+ autodetect=self.autodetect,
  compression=self.compression,
  skip_leading_rows=self.skip_leading_rows,
  field_delimiter=self.field_delimiter,

diff --git a/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst b/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst
@@ -193,6 +193,9 @@ Or you may point the operator to a Google Cloud Storage object name where the sc
  :start-after: [START howto_operator_bigquery_create_table_schema_json]
  :end-before: [END howto_operator_bigquery_create_table_schema_json]
 
+To use BigQuery `schema auto-detection <https://cloud.google.com/bigquery/docs/schema-detect#schema_auto-detection_for_external_data_sources>`__,
+set the ``autodetect`` flag instead of providing explicit schema information.
+
 .. _howto/operator:BigQueryGetDataOperator:
 
 Fetch data from table

diff --git a/tests/providers/google/cloud/operators/test_bigquery.py b/tests/providers/google/cloud/operators/test_bigquery.py
@@ -197,6 +197,7 @@ def test_execute(self, mock_hook):
  bucket=TEST_GCS_BUCKET,
  source_objects=TEST_GCS_DATA,
  source_format=TEST_SOURCE_FORMAT,
+ autodetect=True,
  )
 
  operator.execute(None)
@@ -205,6 +206,7 @@ def test_execute(self, mock_hook):
  schema_fields=[],
  source_uris=[f'gs://{TEST_GCS_BUCKET}/{source_object}' for source_object in TEST_GCS_DATA],
  source_format=TEST_SOURCE_FORMAT,
+ autodetect=True,
  compression='NONE',
  skip_leading_rows=0,
  field_delimiter=',',