diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index e4e943e0..4bdeef39 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:98f3afd11308259de6e828e37376d18867fd321aba07826e29e4f8d9cab56bad -# created: 2024-02-27T15:56:18.442440378Z + digest: sha256:a8a80fc6456e433df53fc2a0d72ca0345db0ddefb409f1b75b118dfd1babd952 +# created: 2024-03-15T16:25:47.905264637Z diff --git a/.kokoro/build.sh b/.kokoro/build.sh index ed749c33..a3cfc6af 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -50,13 +50,6 @@ if [[ -f "${KOKORO_GFILE_DIR}/service-account.json" ]]; then fi -# Remove old nox -python3 -m pip uninstall --yes --quiet nox-automation - -# Install nox -python3 -m pip install --upgrade --quiet nox -python3 -m nox --version - # If this is a continuous build, send the test log to the FlakyBot. # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"continuous"* ]]; then diff --git a/.kokoro/docker/docs/Dockerfile b/.kokoro/docker/docs/Dockerfile index 468b6807..3e385362 100644 --- a/.kokoro/docker/docs/Dockerfile +++ b/.kokoro/docker/docs/Dockerfile @@ -84,4 +84,8 @@ RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \ # Test pip RUN python3 -m pip +# Install build requirements +COPY requirements.txt /requirements.txt +RUN python3 -m pip install --require-hashes -r requirements.txt + CMD ["python3.8"] diff --git a/.kokoro/docker/docs/noxfile.py b/.kokoro/docker/docs/noxfile.py new file mode 100644 index 00000000..483b5590 --- /dev/null +++ b/.kokoro/docker/docs/noxfile.py @@ -0,0 +1,292 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import glob +import os +from pathlib import Path +import sys +from typing import Callable, Dict, Optional + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +BLACK_VERSION = "black==22.3.0" +ISORT_VERSION = "isort==5.10.1" + +# Copy `noxfile_config.py` to your directory and modify it instead. + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + "ignored_versions": [], + # Old samples are opted out of enforcing Python type hints + # All new samples should feature them + "enforce_type_hints": False, + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + "gcloud_project_env": "GOOGLE_CLOUD_PROJECT", + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # If you need to use a specific version of pip, + # change pip_version_override to the string representation + # of the version number, for example, "20.2.4" + "pip_version_override": None, + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + "envs": {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append(".") + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars() -> Dict[str, str]: + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG["gcloud_project_env"] + # This should error out if not set. + ret["GOOGLE_CLOUD_PROJECT"] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG["envs"]) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to test samples. +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False) in ( + "True", + "true", +) + +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + +# +# Style Checks +# + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session: nox.sessions.Session) -> None: + if not TEST_CONFIG["enforce_type_hints"]: + session.install("flake8") + else: + session.install("flake8", "flake8-annotations") + + args = FLAKE8_COMMON_ARGS + [ + ".", + ] + session.run("flake8", *args) + + +# +# Black +# + + +@nox.session +def blacken(session: nox.sessions.Session) -> None: + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + session.run("black", *python_files) + + +# +# format = isort + black +# + +@nox.session +def format(session: nox.sessions.Session) -> None: + """ + Run isort to sort imports. Then run black + to format code to uniform standard. + """ + session.install(BLACK_VERSION, ISORT_VERSION) + python_files = [path for path in os.listdir(".") if path.endswith(".py")] + + # Use the --fss option to sort imports using strict alphabetical order. + # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections + session.run("isort", "--fss", *python_files) + session.run("black", *python_files) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests( + session: nox.sessions.Session, post_install: Callable = None +) -> None: + # check for presence of tests + test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob("**/test_*.py", recursive=True) + test_list.extend(glob.glob("**/tests", recursive=True)) + + if len(test_list) == 0: + print("No tests found, skipping directory.") + return + + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + concurrent_args = [] + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + with open("requirements.txt") as rfile: + packages = rfile.read() + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install( + "-r", "requirements-test.txt", "-c", "constraints-test.txt" + ) + else: + session.install("-r", "requirements-test.txt") + with open("requirements-test.txt") as rtfile: + packages += rtfile.read() + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + if "pytest-parallel" in packages: + concurrent_args.extend(['--workers', 'auto', '--tests-per-worker', 'auto']) + elif "pytest-xdist" in packages: + concurrent_args.extend(['-n', 'auto']) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs + concurrent_args), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session: nox.sessions.Session) -> None: + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip( + "SKIPPED: {} tests are disabled for this sample.".format(session.python) + ) + + +# +# Readmegen +# + + +def _get_repo_root() -> Optional[str]: + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + # .git is not available in repos cloned via Cloud Build + # setup.py is always in the library's root, so use that instead + # https://github.com/googleapis/synthtool/issues/792 + if Path(p / "setup.py").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session: nox.sessions.Session, path: str) -> None: + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/.kokoro/docker/docs/requirements.in b/.kokoro/docker/docs/requirements.in new file mode 100644 index 00000000..816817c6 --- /dev/null +++ b/.kokoro/docker/docs/requirements.in @@ -0,0 +1 @@ +nox diff --git a/.kokoro/docker/docs/requirements.txt b/.kokoro/docker/docs/requirements.txt new file mode 100644 index 00000000..0e5d70f2 --- /dev/null +++ b/.kokoro/docker/docs/requirements.txt @@ -0,0 +1,38 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes requirements.in +# +argcomplete==3.2.3 \ + --hash=sha256:bf7900329262e481be5a15f56f19736b376df6f82ed27576fa893652c5de6c23 \ + --hash=sha256:c12355e0494c76a2a7b73e3a59b09024ca0ba1e279fb9ed6c1b82d5b74b6a70c + # via nox +colorlog==6.8.2 \ + --hash=sha256:3e3e079a41feb5a1b64f978b5ea4f46040a94f11f0e8bbb8261e3dbbeca64d44 \ + --hash=sha256:4dcbb62368e2800cb3c5abd348da7e53f6c362dda502ec27c560b2e58a66bd33 + # via nox +distlib==0.3.8 \ + --hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \ + --hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64 + # via virtualenv +filelock==3.13.1 \ + --hash=sha256:521f5f56c50f8426f5e03ad3b281b490a87ef15bc6c526f168290f0c7148d44e \ + --hash=sha256:57dbda9b35157b05fb3e58ee91448612eb674172fab98ee235ccb0b5bee19a1c + # via virtualenv +nox==2024.3.2 \ + --hash=sha256:e53514173ac0b98dd47585096a55572fe504fecede58ced708979184d05440be \ + --hash=sha256:f521ae08a15adbf5e11f16cb34e8d0e6ea521e0b92868f684e91677deb974553 + # via -r requirements.in +packaging==24.0 \ + --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \ + --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9 + # via nox +platformdirs==4.2.0 \ + --hash=sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068 \ + --hash=sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768 + # via virtualenv +virtualenv==20.25.1 \ + --hash=sha256:961c026ac520bac5f69acb8ea063e8a4f071bcc9457b9c1f28f6b085c511583a \ + --hash=sha256:e08e13ecdca7a0bd53798f356d5831434afa5b07b93f0abdf0797b7a06ffe197 + # via nox diff --git a/.kokoro/requirements.in b/.kokoro/requirements.in index ec867d9f..fff4d9ce 100644 --- a/.kokoro/requirements.in +++ b/.kokoro/requirements.in @@ -1,5 +1,5 @@ gcp-docuploader -gcp-releasetool>=1.10.5 # required for compatibility with cryptography>=39.x +gcp-releasetool>=2 # required for compatibility with cryptography>=42.x importlib-metadata typing-extensions twine @@ -8,3 +8,4 @@ setuptools nox>=2022.11.21 # required to remove dependency on py charset-normalizer<3 click<8.1.0 +cryptography>=42.0.5 diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index bda8e38c..dd61f5f3 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -93,40 +93,41 @@ colorlog==6.7.0 \ # via # gcp-docuploader # nox -cryptography==42.0.4 \ - --hash=sha256:01911714117642a3f1792c7f376db572aadadbafcd8d75bb527166009c9f1d1b \ - --hash=sha256:0e89f7b84f421c56e7ff69f11c441ebda73b8a8e6488d322ef71746224c20fce \ - --hash=sha256:12d341bd42cdb7d4937b0cabbdf2a94f949413ac4504904d0cdbdce4a22cbf88 \ - --hash=sha256:15a1fb843c48b4a604663fa30af60818cd28f895572386e5f9b8a665874c26e7 \ - --hash=sha256:1cdcdbd117681c88d717437ada72bdd5be9de117f96e3f4d50dab3f59fd9ab20 \ - --hash=sha256:1df6fcbf60560d2113b5ed90f072dc0b108d64750d4cbd46a21ec882c7aefce9 \ - --hash=sha256:3c6048f217533d89f2f8f4f0fe3044bf0b2090453b7b73d0b77db47b80af8dff \ - --hash=sha256:3e970a2119507d0b104f0a8e281521ad28fc26f2820687b3436b8c9a5fcf20d1 \ - --hash=sha256:44a64043f743485925d3bcac548d05df0f9bb445c5fcca6681889c7c3ab12764 \ - --hash=sha256:4e36685cb634af55e0677d435d425043967ac2f3790ec652b2b88ad03b85c27b \ - --hash=sha256:5f8907fcf57392cd917892ae83708761c6ff3c37a8e835d7246ff0ad251d9298 \ - --hash=sha256:69b22ab6506a3fe483d67d1ed878e1602bdd5912a134e6202c1ec672233241c1 \ - --hash=sha256:6bfadd884e7280df24d26f2186e4e07556a05d37393b0f220a840b083dc6a824 \ - --hash=sha256:6d0fbe73728c44ca3a241eff9aefe6496ab2656d6e7a4ea2459865f2e8613257 \ - --hash=sha256:6ffb03d419edcab93b4b19c22ee80c007fb2d708429cecebf1dd3258956a563a \ - --hash=sha256:810bcf151caefc03e51a3d61e53335cd5c7316c0a105cc695f0959f2c638b129 \ - --hash=sha256:831a4b37accef30cccd34fcb916a5d7b5be3cbbe27268a02832c3e450aea39cb \ - --hash=sha256:887623fe0d70f48ab3f5e4dbf234986b1329a64c066d719432d0698522749929 \ - --hash=sha256:a0298bdc6e98ca21382afe914c642620370ce0470a01e1bef6dd9b5354c36854 \ - --hash=sha256:a1327f280c824ff7885bdeef8578f74690e9079267c1c8bd7dc5cc5aa065ae52 \ - --hash=sha256:c1f25b252d2c87088abc8bbc4f1ecbf7c919e05508a7e8628e6875c40bc70923 \ - --hash=sha256:c3a5cbc620e1e17009f30dd34cb0d85c987afd21c41a74352d1719be33380885 \ - --hash=sha256:ce8613beaffc7c14f091497346ef117c1798c202b01153a8cc7b8e2ebaaf41c0 \ - --hash=sha256:d2a27aca5597c8a71abbe10209184e1a8e91c1fd470b5070a2ea60cafec35bcd \ - --hash=sha256:dad9c385ba8ee025bb0d856714f71d7840020fe176ae0229de618f14dae7a6e2 \ - --hash=sha256:db4b65b02f59035037fde0998974d84244a64c3265bdef32a827ab9b63d61b18 \ - --hash=sha256:e09469a2cec88fb7b078e16d4adec594414397e8879a4341c6ace96013463d5b \ - --hash=sha256:e53dc41cda40b248ebc40b83b31516487f7db95ab8ceac1f042626bc43a2f992 \ - --hash=sha256:f1e85a178384bf19e36779d91ff35c7617c885da487d689b05c1366f9933ad74 \ - --hash=sha256:f47be41843200f7faec0683ad751e5ef11b9a56a220d57f300376cd8aba81660 \ - --hash=sha256:fb0cef872d8193e487fc6bdb08559c3aa41b659a7d9be48b2e10747f47863925 \ - --hash=sha256:ffc73996c4fca3d2b6c1c8c12bfd3ad00def8621da24f547626bf06441400449 +cryptography==42.0.5 \ + --hash=sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee \ + --hash=sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576 \ + --hash=sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d \ + --hash=sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30 \ + --hash=sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413 \ + --hash=sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb \ + --hash=sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da \ + --hash=sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4 \ + --hash=sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd \ + --hash=sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc \ + --hash=sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8 \ + --hash=sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1 \ + --hash=sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc \ + --hash=sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e \ + --hash=sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8 \ + --hash=sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940 \ + --hash=sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400 \ + --hash=sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7 \ + --hash=sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16 \ + --hash=sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278 \ + --hash=sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74 \ + --hash=sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec \ + --hash=sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1 \ + --hash=sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2 \ + --hash=sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c \ + --hash=sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922 \ + --hash=sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a \ + --hash=sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6 \ + --hash=sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1 \ + --hash=sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e \ + --hash=sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac \ + --hash=sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7 # via + # -r requirements.in # gcp-releasetool # secretstorage distlib==0.3.7 \ @@ -145,9 +146,9 @@ gcp-docuploader==0.6.5 \ --hash=sha256:30221d4ac3e5a2b9c69aa52fdbef68cc3f27d0e6d0d90e220fc024584b8d2318 \ --hash=sha256:b7458ef93f605b9d46a4bf3a8dc1755dad1f31d030c8679edf304e343b347eea # via -r requirements.in -gcp-releasetool==1.16.0 \ - --hash=sha256:27bf19d2e87aaa884096ff941aa3c592c482be3d6a2bfe6f06afafa6af2353e3 \ - --hash=sha256:a316b197a543fd036209d0caba7a8eb4d236d8e65381c80cbc6d7efaa7606d63 +gcp-releasetool==2.0.0 \ + --hash=sha256:3d73480b50ba243f22d7c7ec08b115a30e1c7817c4899781840c26f9c55b8277 \ + --hash=sha256:7aa9fd935ec61e581eb8458ad00823786d91756c25e492f372b2b30962f3c28f # via -r requirements.in google-api-core==2.12.0 \ --hash=sha256:c22e01b1e3c4dcd90998494879612c38d0a3411d1f7b679eb89e2abe3ce1f553 \ @@ -392,29 +393,18 @@ platformdirs==3.11.0 \ --hash=sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3 \ --hash=sha256:e9d171d00af68be50e9202731309c4e658fd8bc76f55c11c7dd760d023bda68e # via virtualenv -protobuf==3.20.3 \ - --hash=sha256:03038ac1cfbc41aa21f6afcbcd357281d7521b4157926f30ebecc8d4ea59dcb7 \ - --hash=sha256:28545383d61f55b57cf4df63eebd9827754fd2dc25f80c5253f9184235db242c \ - --hash=sha256:2e3427429c9cffebf259491be0af70189607f365c2f41c7c3764af6f337105f2 \ - --hash=sha256:398a9e0c3eaceb34ec1aee71894ca3299605fa8e761544934378bbc6c97de23b \ - --hash=sha256:44246bab5dd4b7fbd3c0c80b6f16686808fab0e4aca819ade6e8d294a29c7050 \ - --hash=sha256:447d43819997825d4e71bf5769d869b968ce96848b6479397e29fc24c4a5dfe9 \ - --hash=sha256:67a3598f0a2dcbc58d02dd1928544e7d88f764b47d4a286202913f0b2801c2e7 \ - --hash=sha256:74480f79a023f90dc6e18febbf7b8bac7508420f2006fabd512013c0c238f454 \ - --hash=sha256:819559cafa1a373b7096a482b504ae8a857c89593cf3a25af743ac9ecbd23480 \ - --hash=sha256:899dc660cd599d7352d6f10d83c95df430a38b410c1b66b407a6b29265d66469 \ - --hash=sha256:8c0c984a1b8fef4086329ff8dd19ac77576b384079247c770f29cc8ce3afa06c \ - --hash=sha256:9aae4406ea63d825636cc11ffb34ad3379335803216ee3a856787bcf5ccc751e \ - --hash=sha256:a7ca6d488aa8ff7f329d4c545b2dbad8ac31464f1d8b1c87ad1346717731e4db \ - --hash=sha256:b6cc7ba72a8850621bfec987cb72623e703b7fe2b9127a161ce61e61558ad905 \ - --hash=sha256:bf01b5720be110540be4286e791db73f84a2b721072a3711efff6c324cdf074b \ - --hash=sha256:c02ce36ec760252242a33967d51c289fd0e1c0e6e5cc9397e2279177716add86 \ - --hash=sha256:d9e4432ff660d67d775c66ac42a67cf2453c27cb4d738fc22cb53b5d84c135d4 \ - --hash=sha256:daa564862dd0d39c00f8086f88700fdbe8bc717e993a21e90711acfed02f2402 \ - --hash=sha256:de78575669dddf6099a8a0f46a27e82a1783c557ccc38ee620ed8cc96d3be7d7 \ - --hash=sha256:e64857f395505ebf3d2569935506ae0dfc4a15cb80dc25261176c784662cdcc4 \ - --hash=sha256:f4bd856d702e5b0d96a00ec6b307b0f51c1982c2bf9c0052cf9019e9a544ba99 \ - --hash=sha256:f4c42102bc82a51108e449cbb32b19b180022941c727bac0cfd50170341f16ee +protobuf==4.25.3 \ + --hash=sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4 \ + --hash=sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8 \ + --hash=sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c \ + --hash=sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d \ + --hash=sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4 \ + --hash=sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa \ + --hash=sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c \ + --hash=sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019 \ + --hash=sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9 \ + --hash=sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c \ + --hash=sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2 # via # gcp-docuploader # gcp-releasetool @@ -518,7 +508,7 @@ zipp==3.17.0 \ # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -setuptools==68.2.2 \ - --hash=sha256:4ac1475276d2f1c48684874089fefcd83bd7162ddaafb81fac866ba0db282a87 \ - --hash=sha256:b454a35605876da60632df1a60f736524eb73cc47bbc9f3f1ef1b644de74fd2a +setuptools==69.2.0 \ + --hash=sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e \ + --hash=sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c # via -r requirements.in diff --git a/CHANGELOG.md b/CHANGELOG.md index c1588791..c853f1b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,22 @@ [1]: https://pypi.org/project/google-cloud-ndb/#history +## [2.3.1](https://github.com/googleapis/python-ndb/compare/v2.3.0...v2.3.1) (2024-03-16) + + +### Bug Fixes + +* **grpc:** Fix large payload handling when using the emulator. ([#975](https://github.com/googleapis/python-ndb/issues/975)) ([d9162ae](https://github.com/googleapis/python-ndb/commit/d9162aee709062683bf5f9f01208bd40f46d490a)) +* Remove uses of six. [#913](https://github.com/googleapis/python-ndb/issues/913) ([#958](https://github.com/googleapis/python-ndb/issues/958)) ([e17129a](https://github.com/googleapis/python-ndb/commit/e17129a2114c3f5d45b99cc9a4911b586eb3fafa)) +* Show a non-None error for core_exception.Unknown errors. ([#968](https://github.com/googleapis/python-ndb/issues/968)) ([66e61cc](https://github.com/googleapis/python-ndb/commit/66e61cc578335509d480650906528fa390f44c11)) + + +### Documentation + +* Document how to run system tests against the emulator. ([#963](https://github.com/googleapis/python-ndb/issues/963)) ([47db5b9](https://github.com/googleapis/python-ndb/commit/47db5b9f6ee1fc7c01ad86d476cd8e066fb5cffb)) +* Note to use functools.wrap instead of utils.wrapping. ([#966](https://github.com/googleapis/python-ndb/issues/966)) ([5e9f3d6](https://github.com/googleapis/python-ndb/commit/5e9f3d6977677c20b3447f07bf8bcf4553aac076)) +* Tell users of utils.wrapping to use functools.wraps ([#967](https://github.com/googleapis/python-ndb/issues/967)) ([042645b](https://github.com/googleapis/python-ndb/commit/042645b52608a1c11645dd4b014a90040468b113)) + ## [2.3.0](https://github.com/googleapis/python-ndb/compare/v2.2.2...v2.3.0) (2024-03-01) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 2baa8674..0e13c7b0 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -158,6 +158,11 @@ Running System Tests auth settings and change some configuration in your project to run all the tests. +- System tests may be run against the emulator. To do this, set the + ``DATASTORE_EMULATOR_HOST`` environment variable. Alternatively, + system tests with the emulator can run with + `nox -e emulator-system-PYTHON_VERSION` + - System tests will be run against an actual project and so you'll need to provide some environment variables to facilitate authentication to your project: diff --git a/MIGRATION_NOTES.md b/MIGRATION_NOTES.md index 0264345c..3022b429 100644 --- a/MIGRATION_NOTES.md +++ b/MIGRATION_NOTES.md @@ -284,7 +284,7 @@ refactoring, or new features of Python 3, and are no longer implemented: - `utils.logging_debug()` - `utils.positional()` - `utils.tweak_logging()` -- `utils.wrapping()` +- `utils.wrapping()` (use `functools.wraps` instead) - `utils.threading_local()` ## Bare Metal diff --git a/google/cloud/ndb/_gql.py b/google/cloud/ndb/_gql.py index 60a17075..0b605374 100644 --- a/google/cloud/ndb/_gql.py +++ b/google/cloud/ndb/_gql.py @@ -1,6 +1,5 @@ import datetime import re -import six import time from google.cloud.ndb import context as context_module @@ -656,7 +655,7 @@ def _args_to_val(self, func, args): """ vals = [] for arg in args: - if isinstance(arg, six.string_types + six.integer_types): + if isinstance(arg, (str, int)): val = query_module.Parameter(arg) else: val = arg.Get() @@ -782,7 +781,7 @@ def _raise_cast_error(message): def _time_function(values): if len(values) == 1: value = values[0] - if isinstance(value, six.string_types): + if isinstance(value, str): try: time_tuple = time.strptime(value, "%H:%M:%S") except ValueError as error: @@ -791,7 +790,7 @@ def _time_function(values): ) time_tuple = time_tuple[3:] time_tuple = time_tuple[0:3] - elif isinstance(value, six.integer_types): + elif isinstance(value, int): time_tuple = (value,) else: _raise_cast_error("Invalid argument for time(), {}".format(value)) @@ -808,7 +807,7 @@ def _time_function(values): def _date_function(values): if len(values) == 1: value = values[0] - if isinstance(value, six.string_types): + if isinstance(value, str): try: time_tuple = time.strptime(value, "%Y-%m-%d")[0:6] except ValueError as error: @@ -830,7 +829,7 @@ def _date_function(values): def _datetime_function(values): if len(values) == 1: value = values[0] - if isinstance(value, six.string_types): + if isinstance(value, str): try: time_tuple = time.strptime(value, "%Y-%m-%d %H:%M:%S")[0:6] except ValueError as error: diff --git a/google/cloud/ndb/_retry.py b/google/cloud/ndb/_retry.py index c46a069a..cef5f516 100644 --- a/google/cloud/ndb/_retry.py +++ b/google/cloud/ndb/_retry.py @@ -82,9 +82,10 @@ def retry_wrapper(*args, **kwargs): result = yield result except exceptions.NestedRetryException as e: error = e - except Exception as e: + except BaseException as e: # `e` is removed from locals at end of block error = e # See: https://goo.gl/5J8BMK + if not is_transient_error(error): # If we are in an inner retry block, use special nested # retry exception to bubble up to outer retry. Else, raise @@ -104,6 +105,10 @@ def retry_wrapper(*args, **kwargs): yield tasklets.sleep(sleep_time) + # Unknown errors really want to show up as None, so manually set the error. + if isinstance(error, core_exceptions.Unknown): + error = "google.api_core.exceptions.Unknown" + raise core_exceptions.RetryError( "Maximum number of {} retries exceeded while calling {}".format( retries, callback diff --git a/google/cloud/ndb/client.py b/google/cloud/ndb/client.py index 767b2199..8c2ae578 100644 --- a/google/cloud/ndb/client.py +++ b/google/cloud/ndb/client.py @@ -147,7 +147,14 @@ def __init__( ) if emulator: - channel = grpc.insecure_channel(self.host) + channel = grpc.insecure_channel( + self.host, + options=[ + # Default options provided in DatastoreGrpcTransport, but not when we override the channel. + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) else: user_agent = self.client_info.to_user_agent() channel = _helpers.make_secure_channel( diff --git a/google/cloud/ndb/context.py b/google/cloud/ndb/context.py index ff347660..90a39917 100644 --- a/google/cloud/ndb/context.py +++ b/google/cloud/ndb/context.py @@ -20,7 +20,6 @@ import contextvars import itertools import os -import six import threading import uuid @@ -550,7 +549,7 @@ def set_global_cache_timeout_policy(self, policy): if policy is None: policy = _default_global_cache_timeout_policy - elif isinstance(policy, six.integer_types): + elif isinstance(policy, int): timeout = policy def policy(key): diff --git a/google/cloud/ndb/key.py b/google/cloud/ndb/key.py index 04b1c1ff..7252f9e2 100644 --- a/google/cloud/ndb/key.py +++ b/google/cloud/ndb/key.py @@ -90,7 +90,6 @@ import base64 import functools -import six from google.cloud.datastore import _app_engine_key_pb2 from google.cloud.datastore import key as _key_module @@ -1245,7 +1244,7 @@ def _from_urlsafe(urlsafe, app, namespace, database): Tuple[google.cloud.datastore.key.Key, .Reference]: The key corresponding to ``urlsafe`` and the Reference protobuf. """ - if isinstance(urlsafe, six.string_types): # pragma: NO BRANCH + if isinstance(urlsafe, str): # pragma: NO BRANCH urlsafe = urlsafe.encode("ascii") padding = b"=" * (-len(urlsafe) % 4) urlsafe += padding @@ -1526,7 +1525,7 @@ def _clean_flat_path(flat): if isinstance(kind, type): kind = kind._get_kind() flat[i] = kind - if not isinstance(kind, six.string_types): + if not isinstance(kind, str): raise TypeError( "Key kind must be a string or Model class; " "received {!r}".format(kind) @@ -1537,7 +1536,7 @@ def _clean_flat_path(flat): if id_ is None: if i + 2 < len(flat): raise exceptions.BadArgumentError("Incomplete Key entry must be last") - elif not isinstance(id_, six.string_types + six.integer_types): + elif not isinstance(id_, (str, int)): raise TypeError(_INVALID_ID_TYPE.format(id_)) # Remove trailing ``None`` for a partial key. diff --git a/google/cloud/ndb/model.py b/google/cloud/ndb/model.py index 994daa42..317c99a7 100644 --- a/google/cloud/ndb/model.py +++ b/google/cloud/ndb/model.py @@ -257,7 +257,6 @@ class Person(Model): import inspect import json import pickle -import six import zlib import pytz @@ -1069,7 +1068,7 @@ def _verify_name(name): TypeError: If the ``name`` is not a string. ValueError: If the name contains a ``.``. """ - if not isinstance(name, six.string_types): + if not isinstance(name, str): raise TypeError("Name {!r} is not a string".format(name)) if "." in name: @@ -2102,7 +2101,7 @@ def _legacy_db_get_value(v, p): # If this passes, don't return unicode. except UnicodeDecodeError: try: - sval = six.text_type(sval.decode("utf-8")) + sval = str(sval.decode("utf-8")) except UnicodeDecodeError: pass return sval @@ -2435,7 +2434,7 @@ def _validate(self, value): .BadValueError: If ``value`` is not an :class:`int` or convertible to one. """ - if not isinstance(value, six.integer_types): + if not isinstance(value, int): raise exceptions.BadValueError( "In field {}, expected integer, got {!r}".format(self._name, value) ) @@ -2467,14 +2466,14 @@ def _validate(self, value): .BadValueError: If ``value`` is not a :class:`float` or convertible to one. """ - if not isinstance(value, six.integer_types + (float,)): + if not isinstance(value, (float, int)): raise exceptions.BadValueError( "In field {}, expected float, got {!r}".format(self._name, value) ) return float(value) -class _CompressedValue(six.binary_type): +class _CompressedValue(bytes): """A marker object wrapping compressed values. Args: @@ -2784,7 +2783,7 @@ def _validate(self, value): .BadValueError: If the current property is indexed but the UTF-8 encoded value exceeds the maximum length (1500 bytes). """ - if not isinstance(value, six.text_type): + if not isinstance(value, str): # In Python 2.7, bytes is a synonym for str if isinstance(value, bytes): try: @@ -2811,7 +2810,7 @@ def _to_base_type(self, value): :class:`str`, this will return the UTF-8 encoded bytes for it. Otherwise, it will return :data:`None`. """ - if isinstance(value, six.text_type): + if isinstance(value, str): return value.encode("utf-8") def _from_base_type(self, value): @@ -2946,7 +2945,7 @@ def _validate(self, value): .BadValueError: If the current property is indexed but the UTF-8 encoded value exceeds the maximum length (1500 bytes). """ - if isinstance(value, six.binary_type): + if isinstance(value, bytes): try: encoded_length = len(value) value = value.decode("utf-8") @@ -2956,7 +2955,7 @@ def _validate(self, value): self._name, value ) ) - elif isinstance(value, six.string_types): + elif isinstance(value, str): encoded_length = len(value.encode("utf-8")) else: raise exceptions.BadValueError("Expected string, got {!r}".format(value)) @@ -2978,7 +2977,7 @@ def _to_base_type(self, value): :class:`bytes`, this will return the UTF-8 decoded ``str`` for it. Otherwise, it will return :data:`None`. """ - if isinstance(value, six.binary_type): + if isinstance(value, bytes): return value.decode("utf-8") def _from_base_type(self, value): @@ -3001,7 +3000,7 @@ def _from_base_type(self, value): :class:`str` corresponding to it. Otherwise, it will return :data:`None`. """ - if isinstance(value, six.binary_type): + if isinstance(value, bytes): try: return value.decode("utf-8") except UnicodeError: @@ -3209,7 +3208,7 @@ def _from_base_type(self, value): """ # We write and retrieve `bytes` normally, but for some reason get back # `str` from a projection query. - if not isinstance(value, six.text_type): + if not isinstance(value, str): value = value.decode("ascii") return json.loads(value) @@ -3510,14 +3509,14 @@ def _to_base_type(self, value): user_entity = ds_entity_module.Entity() # Set required fields. - user_entity["email"] = six.ensure_text(value.email()) + user_entity["email"] = str(value.email()) user_entity.exclude_from_indexes.add("email") - user_entity["auth_domain"] = six.ensure_text(value.auth_domain()) + user_entity["auth_domain"] = str(value.auth_domain()) user_entity.exclude_from_indexes.add("auth_domain") # Set optional field. user_id = value.user_id() if user_id: - user_entity["user_id"] = six.ensure_text(user_id) + user_entity["user_id"] = str(user_id) user_entity.exclude_from_indexes.add("user_id") return user_entity @@ -3612,7 +3611,7 @@ def _handle_positional(wrapped): @functools.wraps(wrapped) def wrapper(self, *args, **kwargs): for arg in args: - if isinstance(arg, six.string_types): + if isinstance(arg, str): if "name" in kwargs: raise TypeError("You can only specify name once") @@ -3651,7 +3650,7 @@ def __init__( kind = kind._get_kind() else: - if kind is not None and not isinstance(kind, six.string_types): + if kind is not None and not isinstance(kind, str): raise TypeError("Kind must be a Model class or a string") super(KeyProperty, self).__init__( @@ -3933,7 +3932,7 @@ def _from_base_type(self, value): returns the value without ``tzinfo`` or ``None`` if value did not have ``tzinfo`` set. """ - if isinstance(value, six.integer_types): + if isinstance(value, int): # Projection query, value is integer nanoseconds seconds = value / 1e6 value = datetime.datetime.fromtimestamp(seconds, pytz.utc) @@ -4698,8 +4697,7 @@ def __repr__(cls): return "{}<{}>".format(cls.__name__, ", ".join(props)) -@six.add_metaclass(MetaModel) -class Model(_NotEqualMixin): +class Model(_NotEqualMixin, metaclass=MetaModel): """A class describing Cloud Datastore entities. Model instances are usually called entities. All model classes @@ -4965,7 +4963,7 @@ def __init__(_self, **kwargs): def _get_property_for(self, p, indexed=True, depth=0): """Internal helper to get the Property for a protobuf-level property.""" - if isinstance(p.name(), six.text_type): + if isinstance(p.name(), str): p.set_name(bytes(p.name(), encoding="utf-8")) parts = p.name().decode().split(".") if len(parts) <= depth: @@ -5023,9 +5021,9 @@ def _from_pb(cls, pb, set_key=True, ent=None, key=None): # A key passed in overrides a key in the pb. if key is None and pb.key().path.element_size(): # modern NDB expects strings. - if not isinstance(pb.key_.app_, six.text_type): # pragma: NO BRANCH + if not isinstance(pb.key_.app_, str): # pragma: NO BRANCH pb.key_.app_ = pb.key_.app_.decode() - if not isinstance(pb.key_.name_space_, six.text_type): # pragma: NO BRANCH + if not isinstance(pb.key_.name_space_, str): # pragma: NO BRANCH pb.key_.name_space_ = pb.key_.name_space_.decode() key = Key(reference=pb.key()) @@ -5331,7 +5329,7 @@ def _fix_up_properties(cls): an underscore. """ kind = cls._get_kind() - if not isinstance(kind, six.string_types): + if not isinstance(kind, str): raise KindError( "Class {} defines a ``_get_kind()`` method that returns " "a non-string ({!r})".format(cls.__name__, kind) @@ -6061,7 +6059,7 @@ def _get_or_insert_async(_cls, _name, *args, **kwargs): project = _cls._get_arg(kwargs, "project") options = kwargs.pop("_options") - if not isinstance(name, six.string_types): + if not isinstance(name, str): raise TypeError("'name' must be a string; received {!r}".format(name)) elif not name: @@ -6666,10 +6664,10 @@ def get_indexes(**options): def _unpack_user(v): """Internal helper to unpack a User value from a protocol buffer.""" uv = v.uservalue() - email = six.text_type(uv.email().decode("utf-8")) - auth_domain = six.text_type(uv.auth_domain().decode("utf-8")) + email = str(uv.email().decode("utf-8")) + auth_domain = str(uv.auth_domain().decode("utf-8")) obfuscated_gaiaid = uv.obfuscated_gaiaid().decode("utf-8") - obfuscated_gaiaid = six.text_type(obfuscated_gaiaid) + obfuscated_gaiaid = str(obfuscated_gaiaid) value = User( email=email, diff --git a/google/cloud/ndb/query.py b/google/cloud/ndb/query.py index bc2beadc..76066b75 100644 --- a/google/cloud/ndb/query.py +++ b/google/cloud/ndb/query.py @@ -139,7 +139,6 @@ def ranked(cls, rank): import functools import logging -import six from google.cloud.ndb import context as context_module from google.cloud.ndb import exceptions @@ -306,7 +305,7 @@ class Parameter(ParameterizedThing): """ def __init__(self, key): - if not isinstance(key, six.integer_types + six.string_types): + if not isinstance(key, (int, str)): raise TypeError( "Parameter key must be an integer or string, not {}".format(key) ) @@ -1680,7 +1679,7 @@ def _to_property_orders(self, order_by): elif isinstance(order, model.Property): # use the sign to turn it into a PropertyOrder orders.append(+order) - elif isinstance(order, six.string_types): + elif isinstance(order, str): name = order reverse = False if order.startswith("-"): @@ -2349,7 +2348,7 @@ def _to_property_names(properties): fixed = [] for prop in properties: - if isinstance(prop, six.string_types): + if isinstance(prop, str): fixed.append(prop) elif isinstance(prop, model.Property): fixed.append(prop._name) diff --git a/google/cloud/ndb/utils.py b/google/cloud/ndb/utils.py index 39ceb4e0..a4245320 100644 --- a/google/cloud/ndb/utils.py +++ b/google/cloud/ndb/utils.py @@ -162,4 +162,5 @@ def tweak_logging(*args, **kwargs): def wrapping(*args, **kwargs): + """Use functools.wraps instead""" raise NotImplementedError diff --git a/google/cloud/ndb/version.py b/google/cloud/ndb/version.py index ee9518e1..4be9e8f1 100644 --- a/google/cloud/ndb/version.py +++ b/google/cloud/ndb/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. # -__version__ = "2.3.0" +__version__ = "2.3.1" diff --git a/noxfile.py b/noxfile.py index 2c6bbcb5..cb3c686f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -20,6 +20,8 @@ import os import pathlib import shutil +import signal +import subprocess import nox @@ -91,6 +93,50 @@ def cover(session): session.run("coverage", "erase") +@nox.session(name="emulator-system", python=ALL_INTERPRETERS) +def emulator_system(session): + """Run the system test suite.""" + # Only run the emulator tests manually. + if not session.interactive: + return + + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + system_test_folder_path = os.path.join("tests", "system") + + # Install all test dependencies, then install this package into the + # virtualenv's dist-packages. + session.install("pytest") + session.install("google-cloud-testutils") + for local_dep in LOCAL_DEPS: + session.install(local_dep) + session.install(".", "-c", constraints_path) + + # TODO: It would be better to allow the emulator to bind to any port and pull + # the port from stderr. + emulator_args = [ + "gcloud", + "emulators", + "firestore", + "start", + "--database-mode=datastore-mode", + "--host-port=localhost:8092", + ] + emulator = subprocess.Popen(emulator_args, stderr=subprocess.PIPE) + # Run py.test against the system tests. + session.run( + "py.test", + "--quiet", + system_test_folder_path, + *session.posargs, + env={"DATASTORE_EMULATOR_HOST": "localhost:8092"}, + ) + session.run("curl", "-d", "", "localhost:8092/shutdown", external=True) + emulator.terminate() + emulator.wait(timeout=2) + + def run_black(session, use_check=False): args = ["black"] if use_check: diff --git a/setup.py b/setup.py index d5c32763..c67787af 100644 --- a/setup.py +++ b/setup.py @@ -46,8 +46,6 @@ def main(): "pymemcache >= 2.1.0, < 5.0.0dev", "pytz >= 2018.3", "redis >= 3.0.0, < 6.0.0dev", - # TODO(https://github.com/googleapis/python-ndb/issues/913) remove this dependency once six is no longer used in the codebase - "six >= 1.12.0, < 2.0.0dev" ] setuptools.setup( diff --git a/tests/system/test_crud.py b/tests/system/test_crud.py index 9aeb0960..66d7d1dc 100644 --- a/tests/system/test_crud.py +++ b/tests/system/test_crud.py @@ -401,6 +401,24 @@ def insert(foo): thread2.join() +@pytest.mark.usefixtures("client_context") +def test_large_rpc_lookup(dispose_of, ds_client): + class SomeKind(ndb.Model): + foo = ndb.TextProperty() + + foo = "a" * (500 * 1024) + + keys = [] + for i in range(15): + key = SomeKind(foo=foo).put() + dispose_of(key._key) + keys.append(key) + + retrieved = ndb.get_multi(keys) + for entity in retrieved: + assert entity.foo == foo + + @pytest.mark.usefixtures("client_context") def test_large_json_property(dispose_of, ds_client): class SomeKind(ndb.Model): diff --git a/tests/system/test_query.py b/tests/system/test_query.py index 12bac380..8e40acb3 100644 --- a/tests/system/test_query.py +++ b/tests/system/test_query.py @@ -156,7 +156,7 @@ class SomeKind(ndb.Model): foo = ndb.IntegerProperty() query = SomeKind.query(ancestor=ndb.Key(KIND, root_id)) - results = eventually(query.fetch, length_equals(6)) + results = query.fetch() results = sorted(results, key=operator.attrgetter("foo")) assert [entity.foo for entity in results] == [-1, 0, 1, 2, 3, 4] @@ -180,7 +180,7 @@ class Dummy(ndb.Model): with client_context.new(namespace=other_namespace).use(): query = Dummy.query(ancestor=parent_key, namespace="xyz") - results = eventually(query.fetch, length_equals(2)) + results = query.fetch() assert results[0].foo == "bar" assert results[1].foo == "child" @@ -206,7 +206,7 @@ class Dummy(ndb.Model): with client_context.new(namespace=other_namespace).use(): query = Dummy.query(ancestor=parent_key, namespace="") - results = eventually(query.fetch, length_equals(2)) + results = query.fetch() assert results[0].foo == "bar" assert results[1].foo == "child" diff --git a/tests/unit/test__retry.py b/tests/unit/test__retry.py index 3cb9e1b9..35eddb27 100644 --- a/tests/unit/test__retry.py +++ b/tests/unit/test__retry.py @@ -98,6 +98,18 @@ def callback(): retry = _retry.retry_async(callback) assert retry().exception() is error + @staticmethod + @pytest.mark.usefixtures("in_context") + def test_api_core_unknown(): + def callback(): + raise core_exceptions.Unknown("Unknown") + + with pytest.raises(core_exceptions.RetryError) as e: + retry = _retry.retry_async(callback, retries=1) + retry().result() + + assert e.value.cause == "google.api_core.exceptions.Unknown" + @staticmethod @pytest.mark.usefixtures("in_context") @mock.patch("google.cloud.ndb.tasklets.sleep")