[AIRFLOW-7055] Verbose logging option for google provider (#7711)

apache · Mar 14, 2020 · 2bc020c · 2bc020c
1 parent d466148
commit 2bc020c
Show file tree

Hide file tree

Showing 5 changed files with 90 additions and 1 deletion.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -296,7 +296,7 @@ repos:
  - id: airflow-config-yaml
  name: Checks for consistency between config.yml and default_config.cfg
  language: python
- files: "^airflow/config_templates/config.yml$|^airflow/config_templates/default_airflow.cfg$"
+ files: "config.yml$|default_airflow.cfg$|default.cfg$"
  pass_filenames: false
  require_serial: false
  entry: scripts/ci/pre_commit_yaml_to_cfg.py

diff --git a/airflow/providers/google/__init__.py b/airflow/providers/google/__init__.py
@@ -14,3 +14,19 @@
 # KIND, either express or implied. See the License for the
 # specific language governing permissions and limitations
 # under the License.
+import logging
+
+from airflow.configuration import conf
+
+PROVIDERS_GOOGLE_VERBOSE_LOGGING: bool = conf.getboolean('providers_google',
+ 'VERBOSE_LOGGING', fallback=False)
+if PROVIDERS_GOOGLE_VERBOSE_LOGGING:
+ for logger_name in ["google_auth_httplib2", "httplib2", "googleapiclient"]:
+ logger = logging.getLogger(logger_name)
+ logger.handlers += [handler for handler in
+ logging.getLogger().handlers if handler.name in ["task", "console"]]
+ logger.level = logging.DEBUG
+ logger.propagate = False
+
+ import httplib2
+ httplib2.debuglevel = 4
diff --git a/airflow/providers/google/config_templates/config.yml b/airflow/providers/google/config_templates/config.yml
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+- name: providers_google
+ description: Options for google provider
+ options:
+ - name: verbose_logging
+ description: |
+ Sets verbose logging for google provider
+ version_added: 2.0.0
+ type: boolean
+ example: ~
+ default: "False"
diff --git a/airflow/providers/google/config_templates/default_config.cfg b/airflow/providers/google/config_templates/default_config.cfg
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+# This is the template for Airflow's default configuration. When Airflow is
+# imported, it looks for a configuration file at $AIRFLOW_HOME/airflow.cfg. If
+# it doesn't exist, Airflow uses this template to generate it by replacing
+# variables in curly braces with their global values from configuration.py.
+
+# Users should not modify this file; they should customize the generated
+# airflow.cfg instead.
+
+
+# ----------------------- TEMPLATE BEGINS HERE -----------------------
+
+[providers_google]
+
+# Options for google provider
+# Sets verbose logging for google provider
+verbose_logging = False
diff --git a/scripts/ci/pre_commit_yaml_to_cfg.py b/scripts/ci/pre_commit_yaml_to_cfg.py
@@ -76,6 +76,7 @@ def write_config(yaml_config_file_path: str, default_cfg_file_path: str):
  :param yaml_config_file_path: Full path to config.yaml
  :param default_cfg_file_path: Full path to default_airflow.cfg
  """
+ print(f"Converting {yaml_config_file_path} to {default_cfg_file_path}")
  with open(default_cfg_file_path, 'w') as configfile:
  configfile.writelines(FILE_HEADER)
  config_yaml = read_default_config_yaml(yaml_config_file_path)
@@ -133,3 +134,13 @@ def write_config(yaml_config_file_path: str, default_cfg_file_path: str):
  yaml_config_file_path=airflow_config_yaml_file_path,
  default_cfg_file_path=airflow_default_config_path
  )
+
+ providers_dir = os.path.join(os.path.dirname(__file__), "../../airflow/providers")
+ for root, dir_names, file_names in os.walk(providers_dir):
+ for file_name in file_names:
+ if root.endswith("config_templates") and file_name == 'config.yml' and \
+ os.path.isfile(os.path.join(root, "default_config.cfg")):
+ write_config(
+ yaml_config_file_path=os.path.join(root, "config.yml"),
+ default_cfg_file_path=os.path.join(root, "default_config.cfg")
+ )