Use object instead of array in config.yml for config template (#28417)

This makes it easier to navigate the document in IDE.
apache · Dec 20, 2022 · 820f5a9 · 820f5a9
1 parent 9ac76ec
commit 820f5a9
Show file tree

Hide file tree

Showing 8 changed files with 447 additions and 422 deletions.
diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml
diff --git a/airflow/config_templates/config.yml.schema.json b/airflow/config_templates/config.yml.schema.json
@@ -1,61 +1,87 @@
 {
- "$schema": "http://json-schema.org/draft-07/schema#",
- "type": "array",
- "items": {
+ "$schema": "http://json-schema.org/draft-07/schema#",
  "type": "object",
- "properties": {
- "name": {
- "type": "string"
- },
- "description": {
- "type": ["string", "null"]
- },
- "options": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "name": {
- "type": "string"
- },
+ "additionalProperties": {
+ "type": "object",
+ "properties": {
  "description": {
- "type": ["string", "null"]
- },
- "version_added": {
- "type": ["string", "null"]
- },
- "type": {
- "type": "string",
- "enum": ["string", "boolean", "integer", "float"]
- },
- "example": {
- "type": ["string", "null", "number"]
+ "type": [
+ "string",
+ "null"
+ ]
  },
- "default": {
- "type": ["string", "null", "number"]
+ "options": {
+ "type": "object",
+ "additionalProperties": {
+ "$ref": "#/definitions/option"
+ }
  },
- "sensitive": {
- "type": "boolean",
- "description": "When true, this option is sensitive and can be specified using AIRFLOW__{section}___{name}__SECRET or AIRFLOW__{section}___{name}_CMD environment variables. See: airflow.configuration.AirflowConfigParser.sensitive_config_values"
+ "renamed": {
+ "type": "object",
+ "properties": {
+ "previous_name": {"type": "string"},
+ "version": {"type": "string"}
+ }
  }
- },
- "required": [
- "name",
+ },
+ "required": [
  "description",
- "version_added",
- "type",
- "example",
- "default"
- ],
- "additional_properties": false
- }
- }
+ "options"
+ ],
+ "additionalProperties": false
  },
- "required": [
- "name",
- "description",
- "options"
- ],
- "additional_properties": false
- }
+ "definitions": {
+ "option": {
+ "type": "object",
+ "properties": {
+ "description": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "version_added": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "type": {
+ "type": "string",
+ "enum": [
+ "string",
+ "boolean",
+ "integer",
+ "float"
+ ]
+ },
+ "example": {
+ "type": [
+ "string",
+ "null",
+ "number"
+ ]
+ },
+ "default": {
+ "type": [
+ "string",
+ "null",
+ "number"
+ ]
+ },
+ "sensitive": {
+ "type": "boolean",
+ "description": "When true, this option is sensitive and can be specified using AIRFLOW__{section}___{name}__SECRET or AIRFLOW__{section}___{name}_CMD environment variables. See: airflow.configuration.AirflowConfigParser.sensitive_config_values"
+ }
+ },
+ "required": [
+ "description",
+ "version_added",
+ "type",
+ "example",
+ "default"
+ ],
+ "additional_properties": false
+ }
+ }
 }
diff --git a/airflow/configuration.py b/airflow/configuration.py
@@ -136,7 +136,7 @@ def _default_config_file_path(file_name: str) -> str:
  return os.path.join(templates_dir, file_name)
 
 
-def default_config_yaml() -> list[dict[str, Any]]:
+def default_config_yaml() -> dict[str, Any]:
  """
  Read Airflow configs from YAML file.
 

diff --git a/airflow/providers/google/config_templates/config.yml b/airflow/providers/google/config_templates/config.yml
@@ -15,10 +15,10 @@
 # specific language governing permissions and limitations
 # under the License.
 ---
-- name: providers_google
+providers_google:
  description: Options for google provider
  options:
- - name: verbose_logging
+ verbose_logging:
  description: |
  Sets verbose logging for google provider
  version_added: 2.0.0

diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -325,7 +325,7 @@ def build_docs(
 ):
  """Build documentation in the container."""
  if for_production and not clean_build:
- get_console().print("\n[warning]When building docs for production, clan-build is forced\n")
+ get_console().print("\n[warning]When building docs for production, clean-build is forced\n")
  clean_build = True
  perform_environment_checks()
  cleanup_python_generated_files()

diff --git a/docs/apache-airflow/configurations-ref.rst b/docs/apache-airflow/configurations-ref.rst
@@ -41,12 +41,12 @@ that you run airflow components on is synchronized (for example using ntpd) othe
 
 .. jinja:: config_ctx
 
- {% for section in configs %}
+ {% for section_name, section in configs.items() %}
 
- .. _config:{{ section["name"] }}:
+ .. _config:{{ section_name }}:
 
- [{{ section["name"] }}]
- {{ "=" * (section["name"]|length + 2) }}
+ [{{ section_name }}]
+ {{ "=" * (section_name|length + 2) }}
 
  {% if 'renamed' in section %}
  *Renamed in version {{ section['renamed']['version'] }}, previous name was {{ section['renamed']['previous_name'] }}*
@@ -56,12 +56,12 @@ that you run airflow components on is synchronized (for example using ntpd) othe
  {{ section["description"] }}
  {% endif %}
 
- {% for option in section["options"] %}
+ {% for option_name, option in section["options"].items() %}
 
- .. _config:{{ section["name"] }}__{{ option["name"] }}:
+ .. _config:{{ section_name }}__{{ option_name }}:
 
- {{ option["name"] }}
- {{ "-" * option["name"]|length }}
+ {{ option_name }}
+ {{ "-" * option_name|length }}
 
  {% if option["version_added"] %}
  .. versionadded:: {{ option["version_added"] }}
@@ -79,13 +79,13 @@ that you run airflow components on is synchronized (for example using ntpd) othe
  :Default: ``{{ "''" if option["default"] == "" else option["default"] }}``
  {% if option.get("sensitive") %}
  :Environment Variables:
- ``AIRFLOW__{{ section["name"] | upper }}__{{ option["name"] | upper }}``
+ ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}``
 
- ``AIRFLOW__{{ section["name"] | upper }}__{{ option["name"] | upper }}_CMD``
+ ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}_CMD``
 
- ``AIRFLOW__{{ section["name"] | upper }}__{{ option["name"] | upper }}_SECRET``
+ ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}_SECRET``
  {% else %}
- :Environment Variable: ``AIRFLOW__{{ section["name"] | upper }}__{{ option["name"] | upper }}``
+ :Environment Variable: ``AIRFLOW__{{ section_name | upper }}__{{ option_name | upper }}``
  {% endif %}
  {% if option["example"] %}
  :Example:
@@ -94,10 +94,10 @@ that you run airflow components on is synchronized (for example using ntpd) othe
 
  {% endfor %}
 
- {% if section["name"] in deprecated_options %}
+ {% if section_name in deprecated_options %}
 
- {% for deprecated_option_name, (new_section_name, new_option_name, since_version) in deprecated_options[section["name"]].items() %}
- .. _config:{{ section["name"] }}__{{ deprecated_option_name }}:
+ {% for deprecated_option_name, (new_section_name, new_option_name, since_version) in deprecated_options[section_name].items() %}
+ .. _config:{{ section_name }}__{{ deprecated_option_name }}:
 
  {{ deprecated_option_name }} (Deprecated)
  {{ "-" * (deprecated_option_name + " (Deprecated)")|length }}

diff --git a/docs/conf.py b/docs/conf.py
@@ -397,15 +397,15 @@ def _get_rst_filepath_from_path(filepath: pathlib.Path):
  # the config has been templated, not before
  # e.g. {{dag_id}} in default_config.cfg -> {dag_id} in airflow.cfg, and what we want in docs
  keys_to_format = ["default", "example"]
- for conf_section in configs:
- for option in conf_section["options"]:
+ for conf_name, conf_section in configs.items():
+ for option_name, option in conf_section["options"].items():
  for key in keys_to_format:
  if option[key] and "{{" in option[key]:
  option[key] = option[key].replace("{{", "{").replace("}}", "}")
  # Sort options, config and deprecated options for JINJA variables to display
- for config in configs:
- config["options"] = sorted(config["options"], key=lambda o: o["name"])
- configs = sorted(configs, key=lambda l: l["name"])
+ for section_name, config in configs.items():
+ config["options"] = {k: v for k, v in sorted(config["options"].items())}
+ configs = {k: v for k, v in sorted(configs.items())}
  for section in deprecated_options:
  deprecated_options[section] = {k: v for k, v in sorted(deprecated_options[section].items())}
 

diff --git a/scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py b/scripts/ci/pre_commit/pre_commit_yaml_to_cfg.py
@@ -81,12 +81,11 @@ def write_config(yaml_config_file_path: str, default_cfg_file_path: str):
  configfile.writelines(FILE_HEADER)
  config_yaml = read_default_config_yaml(yaml_config_file_path)
 
- for section in config_yaml:
- _write_section(configfile, section)
+ for section_name, section in config_yaml.items():
+ _write_section(configfile, section_name, section)
 
 
-def _write_section(configfile, section):
- section_name = section["name"]
+def _write_section(configfile, section_name, section):
  configfile.write(f"\n[{section_name}]\n")
  section_description = None
  if section["description"] is not None:
@@ -100,11 +99,11 @@ def _write_section(configfile, section):
  configfile.write("#\n")
  else:
  configfile.write(f"# {single_line_desc}\n")
- for idx, option in enumerate(section["options"]):
- _write_option(configfile, idx, option)
+ for idx, (option_name, option) in enumerate(section["options"].items()):
+ _write_option(configfile, idx, option_name, option)
 
 
-def _write_option(configfile, idx, option):
+def _write_option(configfile, idx, option_name, option):
  option_description = None
  if option["description"] is not None:
  option_description = list(filter(lambda x: x is not None, option["description"].splitlines()))
@@ -119,24 +118,24 @@ def _write_option(configfile, idx, option):
  configfile.write(f"# {single_line_desc}\n")
 
  if option["example"]:
- if not str(option["name"]).endswith("_template"):
+ if not str(option_name).endswith("_template"):
  option["example"] = option["example"].replace("{", "{{").replace("}", "}}")
- configfile.write(f"# Example: {option['name']} = {option['example']}\n")
+ configfile.write(f"# Example: {option_name} = {option['example']}\n")
 
  if option["default"] is not None:
  if not isinstance(option["default"], str):
  raise Exception(
- f"Key \"default\" in element with name=\"{option['name']}\" has an invalid type. "
+ f'Key "default" in element with name="{option_name}" has an invalid type. '
  f"Current type: {type(option['default'])}"
  )
  # Remove trailing whitespace on empty string
  if option["default"]:
  value = " " + option["default"]
  else:
  value = ""
- configfile.write(f"{option['name']} ={value}\n")
+ configfile.write(f"{option_name} ={value}\n")
  else:
- configfile.write(f"# {option['name']} =\n")
+ configfile.write(f"# {option_name} =\n")
 
 
 if __name__ == "__main__":