Consolidate importing of os.path.* (#34060)

Co-authored-by: Tzu-ping Chung <[email protected]>
apache · Sep 7, 2023 · 9079093 · 9079093
1 parent a61b5e8
commit 9079093
Show file tree

Hide file tree

Showing 20 changed files with 47 additions and 56 deletions.
diff --git a/airflow/providers/amazon/aws/hooks/base_aws.py b/airflow/providers/amazon/aws/hooks/base_aws.py
@@ -32,7 +32,6 @@
 import warnings
 from copy import deepcopy
 from functools import cached_property, wraps
-from os import PathLike
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, Generic, TypeVar, Union
 
@@ -819,7 +818,7 @@ def test_connection(self):
  return False, str(f"{type(e).__name__!r} error occurred while testing connection: {e}")
 
  @cached_property
- def waiter_path(self) -> PathLike[str] | None:
+ def waiter_path(self) -> os.PathLike[str] | None:
  filename = self.client_type if self.client_type else self.resource_type
  path = Path(__file__).parents[1].joinpath(f"waiters/{filename}.json").resolve()
  return path if path.exists() else None

diff --git a/airflow/providers/amazon/aws/operators/glue.py b/airflow/providers/amazon/aws/operators/glue.py
@@ -17,7 +17,7 @@
 # under the License.
 from __future__ import annotations
 
-import os.path
+import os
 import urllib.parse
 from functools import cached_property
 from typing import TYPE_CHECKING, Sequence

diff --git a/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py b/airflow/providers/amazon/aws/transfers/dynamodb_to_s3.py
@@ -19,11 +19,11 @@
 from __future__ import annotations
 
 import json
+import os
 from copy import copy
 from datetime import datetime
 from decimal import Decimal
 from functools import cached_property
-from os.path import getsize
 from tempfile import NamedTemporaryFile
 from typing import IO, TYPE_CHECKING, Any, Callable, Sequence
 from uuid import uuid4
@@ -197,7 +197,7 @@ def _scan_dynamodb_and_upload_to_s3(self, temp_file: IO, scan_kwargs: dict, tabl
  scan_kwargs["ExclusiveStartKey"] = last_evaluated_key
 
  # Upload the file to S3 if reach file size limit
- if getsize(temp_file.name) >= self.file_size:
+ if os.path.getsize(temp_file.name) >= self.file_size:
  _upload_file_to_s3(temp_file, self.s3_bucket_name, self.s3_key_prefix, self.dest_aws_conn_id)
  temp_file.close()
 

diff --git a/airflow/providers/ftp/hooks/ftp.py b/airflow/providers/ftp/hooks/ftp.py
@@ -19,7 +19,7 @@
 
 import datetime
 import ftplib
-import os.path
+import os
 from typing import Any, Callable
 
 from airflow.hooks.base import BaseHook

diff --git a/airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py b/airflow/providers/google/cloud/example_dags/example_cloud_sql_query.py
@@ -40,7 +40,7 @@
 import os
 import subprocess
 from datetime import datetime
-from os.path import expanduser
+from pathlib import Path
 from urllib.parse import quote_plus
 
 from airflow import models
@@ -87,17 +87,14 @@
 
 # [START howto_operator_cloudsql_query_connections]
 
-HOME_DIR = expanduser("~")
+HOME_DIR = Path.home()
 
 
 def get_absolute_path(path):
  """
  Returns absolute path.
  """
- if path.startswith("/"):
- return path
- else:
- return os.path.join(HOME_DIR, path)
+ return os.fspath(HOME_DIR / path)
 
 
 postgres_kwargs = {

diff --git a/airflow/providers/google/cloud/hooks/cloud_sql.py b/airflow/providers/google/cloud/hooks/cloud_sql.py
@@ -21,7 +21,6 @@
 import errno
 import json
 import os
-import os.path
 import platform
 import random
 import re

diff --git a/airflow/providers/google/cloud/hooks/gcs.py b/airflow/providers/google/cloud/hooks/gcs.py
@@ -28,7 +28,6 @@
 from contextlib import contextmanager
 from functools import partial
 from io import BytesIO
-from os import path
 from tempfile import NamedTemporaryFile
 from typing import IO, TYPE_CHECKING, Any, Callable, Generator, Sequence, TypeVar, cast, overload
 from urllib.parse import urlsplit
@@ -1290,7 +1289,7 @@ def _calculate_sync_destination_path(
  self, blob: storage.Blob, destination_object: str | None, source_object_prefix_len: int
  ) -> str:
  return (
- path.join(destination_object, blob.name[source_object_prefix_len:])
+ os.path.join(destination_object, blob.name[source_object_prefix_len:])
  if destination_object
  else blob.name[source_object_prefix_len:]
  )

diff --git a/airflow/providers/google/cloud/utils/credentials_provider.py b/airflow/providers/google/cloud/utils/credentials_provider.py
@@ -21,7 +21,7 @@
 
 import json
 import logging
-import os.path
+import os
 import tempfile
 from contextlib import ExitStack, contextmanager
 from typing import Collection, Generator, Sequence

diff --git a/airflow/www/extensions/init_views.py b/airflow/www/extensions/init_views.py
@@ -17,9 +17,10 @@
 from __future__ import annotations
 
 import logging
+import os
 import warnings
 from functools import cached_property
-from os import path
+from pathlib import Path
 from typing import TYPE_CHECKING
 
 from connexion import FlaskApi, ProblemException, Resolver
@@ -39,7 +40,7 @@
 log = logging.getLogger(__name__)
 
 # airflow/www/extensions/init_views.py => airflow/
-ROOT_APP_DIR = path.abspath(path.join(path.dirname(__file__), path.pardir, path.pardir))
+ROOT_APP_DIR = Path(__file__).parents[2].resolve()
 
 
 def init_flash_views(app):
@@ -253,15 +254,15 @@ def _handle_method_not_allowed(ex):
  else:
  return views.method_not_allowed(ex)
 
- with open(path.join(ROOT_APP_DIR, "api_connexion", "openapi", "v1.yaml")) as f:
+ with ROOT_APP_DIR.joinpath("api_connexion", "openapi", "v1.yaml").open() as f:
  specification = safe_load(f)
  api_bp = FlaskApi(
  specification=specification,
  resolver=_LazyResolver(),
  base_path=base_path,
  options={
  "swagger_ui": conf.getboolean("webserver", "enable_swagger_ui", fallback=True),
- "swagger_path": path.join(ROOT_APP_DIR, "www", "static", "dist", "swagger-ui"),
+ "swagger_path": os.fspath(ROOT_APP_DIR.joinpath("www", "static", "dist", "swagger-ui")),
  },
  strict_validation=True,
  validate_responses=True,
@@ -279,7 +280,7 @@ def init_api_internal(app: Flask, standalone_api: bool = False) -> None:
  if not standalone_api and not conf.getboolean("webserver", "run_internal_api", fallback=False):
  return
 
- with open(path.join(ROOT_APP_DIR, "api_internal", "openapi", "internal_api_v1.yaml")) as f:
+ with ROOT_APP_DIR.joinpath("api_internal", "openapi", "internal_api_v1.yaml").open() as f:
  specification = safe_load(f)
  api_bp = FlaskApi(
  specification=specification,

diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py
@@ -37,7 +37,6 @@
 from datetime import datetime, timedelta
 from enum import Enum
 from functools import lru_cache
-from os.path import dirname, relpath
 from pathlib import Path
 from random import choice
 from shutil import copyfile
@@ -261,7 +260,7 @@ def get_target_folder() -> str:
 
  :return: the folder path
  """
- return os.path.abspath(os.path.join(dirname(__file__), os.pardir, os.pardir, "provider_packages"))
+ return os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "provider_packages"))
 
 
 def get_target_providers_folder() -> str:
@@ -1154,7 +1153,7 @@ def get_provider_jinja_context(
  "PIP_REQUIREMENTS_TABLE": pip_requirements_table,
  "PIP_REQUIREMENTS_TABLE_RST": pip_requirements_table_rst,
  "PROVIDER_INFO": provider_info,
- "CHANGELOG_RELATIVE_PATH": relpath(
+ "CHANGELOG_RELATIVE_PATH": os.path.relpath(
  provider_details.source_provider_package_path,
  provider_details.documentation_provider_package_path,
  ),

diff --git a/docs/exts/exampleinclude.py b/docs/exts/exampleinclude.py
@@ -20,8 +20,8 @@
 from __future__ import annotations
 
 """Nice formatted include for examples"""
+import os
 import traceback
-from os import path
 
 from docutils import nodes
 
@@ -227,8 +227,8 @@ def doctree_read(app, doctree):
 
  for objnode in doctree.traverse(ExampleHeader):
  filepath = objnode.get("filename")
- relative_path = path.relpath(
- filepath, path.commonprefix([app.config.exampleinclude_sourceroot, filepath])
+ relative_path = os.path.relpath(
+ filepath, os.path.commonprefix([app.config.exampleinclude_sourceroot, filepath])
  )
  modname = relative_path.replace("/", ".")[:-3]
  show_button = register_source(app, env, modname)

diff --git a/scripts/ci/pre_commit/pre_commit_check_order_setup.py b/scripts/ci/pre_commit/pre_commit_check_order_setup.py
@@ -26,15 +26,14 @@
 import re
 import sys
 import textwrap
-from os.path import abspath, dirname
+from pathlib import Path
 
 from rich import print
 
 errors: list[str] = []
 
-MY_DIR_PATH = os.path.dirname(__file__)
-SOURCE_DIR_PATH = os.path.abspath(os.path.join(MY_DIR_PATH, os.pardir, os.pardir, os.pardir))
-sys.path.insert(0, SOURCE_DIR_PATH)
+SOURCE_DIR_PATH = Path(__file__).parents[3].resolve()
+sys.path.insert(0, os.fspath(SOURCE_DIR_PATH))
 
 
 class ConsoleDiff(difflib.Differ):
@@ -124,7 +123,7 @@ def check_install_and_setup_requires() -> None:
 
  from setuptools.config import read_configuration
 
- path = abspath(os.path.join(dirname(__file__), os.pardir, os.pardir, os.pardir, "setup.cfg"))
+ path = os.fspath(SOURCE_DIR_PATH / "setup.cfg")
  config = read_configuration(path)
 
  pattern_dependent_version = re.compile("[~|><=;].*")

diff --git a/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py b/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
@@ -24,18 +24,18 @@
 import os
 import re
 import sys
-from os.path import dirname
+from pathlib import Path
 
 from rich import print
 from rich.console import Console
 from rich.table import Table
 
-AIRFLOW_SOURCES_DIR = os.path.join(dirname(__file__), os.pardir, os.pardir, os.pardir)
+AIRFLOW_SOURCES_DIR = Path(__file__).parents[3].resolve()
 SETUP_PY_FILE = "setup.py"
 DOCS_FILE = os.path.join("docs", "apache-airflow", "extra-packages-ref.rst")
 PY_IDENTIFIER = r"[a-zA-Z_][a-zA-Z0-9_\.]*"
 
-sys.path.insert(0, AIRFLOW_SOURCES_DIR)
+sys.path.insert(0, os.fspath(AIRFLOW_SOURCES_DIR))
 
 os.environ["_SKIP_PYTHON_VERSION_CHECK"] = "true"
 
@@ -49,9 +49,8 @@
 
 
 def get_file_content(*path_elements: str) -> str:
- file_path = os.path.join(AIRFLOW_SOURCES_DIR, *path_elements)
- with open(file_path) as file_to_read:
- return file_to_read.read()
+ file_path = AIRFLOW_SOURCES_DIR.joinpath(*path_elements)
+ return file_path.read_text()
 
 
 def get_extras_from_setup() -> set[str]:

diff --git a/scripts/ci/pre_commit/pre_commit_inline_scripts_in_docker.py b/scripts/ci/pre_commit/pre_commit_inline_scripts_in_docker.py
@@ -17,7 +17,6 @@
 # under the License.
 from __future__ import annotations
 
-from os import listdir
 from pathlib import Path
 
 AIRFLOW_SOURCES_DIR = Path(__file__).parents[3].resolve()
@@ -45,16 +44,16 @@ def insert_content(file_path: Path, content: list[str], header: str, footer: str
  SCRIPTS_DOCKER_DIR = AIRFLOW_SOURCES_DIR / "scripts" / "docker"
 
  for file in [DOCKERFILE_FILE, DOCKERFILE_CI_FILE]:
- for script in listdir(SCRIPTS_DOCKER_DIR):
- script_content = (SCRIPTS_DOCKER_DIR / script).read_text().splitlines(keepends=True)
+ for script in SCRIPTS_DOCKER_DIR.iterdir():
+ script_content = script.read_text().splitlines(keepends=True)
  no_comments_script_content = [
  line for line in script_content if not line.startswith("#") or line.startswith("#!")
  ]
- no_comments_script_content.insert(0, f'COPY <<"EOF" /{script}\n')
+ no_comments_script_content.insert(0, f'COPY <<"EOF" /{script.name}\n')
  insert_content(
  file_path=file,
  content=no_comments_script_content,
  header="# The content below is automatically copied from scripts/docker/",
  footer="EOF",
- file_name=script,
+ file_name=script.name,
  )
diff --git a/scripts/in_container/update_quarantined_test_status.py b/scripts/in_container/update_quarantined_test_status.py
@@ -21,7 +21,7 @@
 import re
 import sys
 from datetime import datetime
-from os.path import dirname, join, realpath
+from pathlib import Path
 from typing import NamedTuple
 from urllib.parse import urlsplit
 
@@ -234,7 +234,7 @@ def get_table(history_map: dict[str, TestHistory]) -> str:
  print()
  print(table)
  print()
- with open(join(dirname(realpath(__file__)), "quarantine_issue_header.md")) as f:
+ with Path(__file__).resolve().with_name("quarantine_issue_header.md").open() as f:
  header = jinja2.Template(f.read(), autoescape=True, undefined=StrictUndefined).render(
  DATE_UTC_NOW=datetime.utcnow()
  )

diff --git a/setup.py b/setup.py
@@ -30,7 +30,6 @@
 import sys
 import unittest
 from copy import deepcopy
-from os.path import relpath
 from pathlib import Path
 from textwrap import wrap
 from typing import Iterable
@@ -864,7 +863,9 @@ def parse_config_files(self, *args, **kwargs) -> None:
  ]
  provider_yaml_files = glob.glob("airflow/providers/**/provider.yaml", recursive=True)
  for provider_yaml_file in provider_yaml_files:
- provider_relative_path = relpath(provider_yaml_file, str(AIRFLOW_SOURCES_ROOT / "airflow"))
+ provider_relative_path = os.path.relpath(
+ provider_yaml_file, str(AIRFLOW_SOURCES_ROOT / "airflow")
+ )
  self.package_data["airflow"].append(provider_relative_path)
  else:
  self.install_requires.extend(

diff --git a/tests/providers/google/cloud/operators/test_dataprep_system.py b/tests/providers/google/cloud/operators/test_dataprep_system.py
@@ -18,7 +18,7 @@
 from __future__ import annotations
 
 import json
-from os import environ
+import os
 
 import pytest
 
@@ -27,7 +27,7 @@
 from tests.test_utils.db import clear_db_connections
 from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest
 
-TOKEN = environ.get("DATAPREP_TOKEN")
+TOKEN = os.environ.get("DATAPREP_TOKEN")
 EXTRA = {"token": TOKEN}
 
 

diff --git a/tests/system/providers/amazon/aws/example_google_api_sheets_to_s3.py b/tests/system/providers/amazon/aws/example_google_api_sheets_to_s3.py
@@ -20,8 +20,8 @@
 """
 from __future__ import annotations
 
+import os
 from datetime import datetime
-from os import getenv
 
 from airflow import DAG
 from airflow.models.baseoperator import chain
@@ -34,9 +34,9 @@
 
 DAG_ID = "example_google_api_sheets_to_s3"
 
-GOOGLE_SHEET_ID = getenv("GOOGLE_SHEET_ID", "test-google-sheet-id")
-GOOGLE_SHEET_RANGE = getenv("GOOGLE_SHEET_RANGE", "test-google-sheet-range")
-S3_DESTINATION_KEY = getenv("S3_DESTINATION_KEY", "s3://test-bucket/key.json")
+GOOGLE_SHEET_ID = os.getenv("GOOGLE_SHEET_ID", "test-google-sheet-id")
+GOOGLE_SHEET_RANGE = os.getenv("GOOGLE_SHEET_RANGE", "test-google-sheet-range")
+S3_DESTINATION_KEY = os.getenv("S3_DESTINATION_KEY", "s3://test-bucket/key.json")
 
 with DAG(
  dag_id=DAG_ID,