From b359a9a55936a759a36aa69c5e5b014685e1fca6 Mon Sep 17 00:00:00 2001 From: Lingqing Gan Date: Mon, 4 Mar 2024 11:06:07 -0800 Subject: [PATCH 01/10] feat: support RANGE query parameters (#1827) * feat: RANGE query parameters and unit tests * unit test * unit test coverage * lint * lint * lint * system test * fix system test * ajust init items order * fix typos and improve docstrings --- benchmark/benchmark.py | 2 +- google/cloud/bigquery/__init__.py | 4 + google/cloud/bigquery/query.py | 297 ++++++++++++++++ tests/system/test_query.py | 33 ++ tests/unit/test_query.py | 548 ++++++++++++++++++++++++++++++ 5 files changed, 883 insertions(+), 1 deletion(-) diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py index 30e294baa..d7dc78678 100644 --- a/benchmark/benchmark.py +++ b/benchmark/benchmark.py @@ -231,7 +231,7 @@ def _is_datetime_min(time_str: str) -> bool: def _summary(run: dict) -> str: - """Coverts run dict to run summary string.""" + """Converts run dict to run summary string.""" no_val = "NODATA" output = ["QUERYTIME "] diff --git a/google/cloud/bigquery/__init__.py b/google/cloud/bigquery/__init__.py index 1ea056eb8..caf81d9aa 100644 --- a/google/cloud/bigquery/__init__.py +++ b/google/cloud/bigquery/__init__.py @@ -83,6 +83,8 @@ from google.cloud.bigquery.query import ConnectionProperty from google.cloud.bigquery.query import ScalarQueryParameter from google.cloud.bigquery.query import ScalarQueryParameterType +from google.cloud.bigquery.query import RangeQueryParameter +from google.cloud.bigquery.query import RangeQueryParameterType from google.cloud.bigquery.query import SqlParameterScalarTypes from google.cloud.bigquery.query import StructQueryParameter from google.cloud.bigquery.query import StructQueryParameterType @@ -122,10 +124,12 @@ "ArrayQueryParameter", "ScalarQueryParameter", "StructQueryParameter", + "RangeQueryParameter", "ArrayQueryParameterType", "ScalarQueryParameterType", "SqlParameterScalarTypes", "StructQueryParameterType", + "RangeQueryParameterType", # Datasets "Dataset", "DatasetReference", diff --git a/google/cloud/bigquery/query.py b/google/cloud/bigquery/query.py index a06ece503..9c9402b74 100644 --- a/google/cloud/bigquery/query.py +++ b/google/cloud/bigquery/query.py @@ -30,6 +30,8 @@ Union[str, int, float, decimal.Decimal, bool, datetime.datetime, datetime.date] ] +_RANGE_ELEMENT_TYPE_STR = {"TIMESTAMP", "DATETIME", "DATE"} + class ConnectionProperty: """A connection-level property to customize query behavior. @@ -362,6 +364,129 @@ def __repr__(self): return f"{self.__class__.__name__}({items}{name}{description})" +class RangeQueryParameterType(_AbstractQueryParameterType): + """Type representation for range query parameters. + + Args: + type_ (Union[ScalarQueryParameterType, str]): + Type of range element, must be one of 'TIMESTAMP', 'DATETIME', or + 'DATE'. + name (Optional[str]): + The name of the query parameter. Primarily used if the type is + one of the subfields in ``StructQueryParameterType`` instance. + description (Optional[str]): + The query parameter description. Primarily used if the type is + one of the subfields in ``StructQueryParameterType`` instance. + """ + + @classmethod + def _parse_range_element_type(self, type_): + """Helper method that parses the input range element type, which may + be a string, or a ScalarQueryParameterType object. + + Returns: + google.cloud.bigquery.query.ScalarQueryParameterType: Instance + """ + if isinstance(type_, str): + if type_ not in _RANGE_ELEMENT_TYPE_STR: + raise ValueError( + "If given as a string, range element type must be one of " + "'TIMESTAMP', 'DATE', or 'DATETIME'." + ) + return ScalarQueryParameterType(type_) + elif isinstance(type_, ScalarQueryParameterType): + if type_._type not in _RANGE_ELEMENT_TYPE_STR: + raise ValueError( + "If given as a ScalarQueryParameter object, range element " + "type must be one of 'TIMESTAMP', 'DATE', or 'DATETIME' " + "type." + ) + return type_ + else: + raise ValueError( + "range_type must be a string or ScalarQueryParameter object, " + "of 'TIMESTAMP', 'DATE', or 'DATETIME' type." + ) + + def __init__(self, type_, *, name=None, description=None): + self.type_ = self._parse_range_element_type(type_) + self.name = name + self.description = description + + @classmethod + def from_api_repr(cls, resource): + """Factory: construct parameter type from JSON resource. + + Args: + resource (Dict): JSON mapping of parameter + + Returns: + google.cloud.bigquery.query.RangeQueryParameterType: Instance + """ + type_ = resource["rangeElementType"]["type"] + name = resource.get("name") + description = resource.get("description") + + return cls(type_, name=name, description=description) + + def to_api_repr(self): + """Construct JSON API representation for the parameter type. + + Returns: + Dict: JSON mapping + """ + # Name and description are only used if the type is a field inside a struct + # type, but it's StructQueryParameterType's responsibilty to use these two + # attributes in the API representation when needed. Here we omit them. + return { + "type": "RANGE", + "rangeElementType": self.type_.to_api_repr(), + } + + def with_name(self, new_name: Union[str, None]): + """Return a copy of the instance with ``name`` set to ``new_name``. + + Args: + name (Union[str, None]): + The new name of the range query parameter type. If ``None``, + the existing name is cleared. + + Returns: + google.cloud.bigquery.query.RangeQueryParameterType: + A new instance with updated name. + """ + return type(self)(self.type_, name=new_name, description=self.description) + + def __repr__(self): + name = f", name={self.name!r}" if self.name is not None else "" + description = ( + f", description={self.description!r}" + if self.description is not None + else "" + ) + return f"{self.__class__.__name__}({self.type_!r}{name}{description})" + + def _key(self): + """A tuple key that uniquely describes this field. + + Used to compute this instance's hashcode and evaluate equality. + + Returns: + Tuple: The contents of this + :class:`~google.cloud.bigquery.query.RangeQueryParameterType`. + """ + type_ = self.type_.to_api_repr() + return (self.name, type_, self.description) + + def __eq__(self, other): + if not isinstance(other, RangeQueryParameterType): + return NotImplemented + return self._key() == other._key() + + def __ne__(self, other): + return not self == other + + class _AbstractQueryParameter(object): """Base class for named / positional query parameters.""" @@ -811,6 +936,178 @@ def __repr__(self): return "StructQueryParameter{}".format(self._key()) +class RangeQueryParameter(_AbstractQueryParameter): + """Named / positional query parameters for range values. + + Args: + range_element_type (Union[str, RangeQueryParameterType]): + The type of range elements. It must be one of 'TIMESTAMP', + 'DATE', or 'DATETIME'. + + start (Optional[Union[ScalarQueryParameter, str]]): + The start of the range value. Must be the same type as + range_element_type. If not provided, it's interpreted as UNBOUNDED. + + end (Optional[Union[ScalarQueryParameter, str]]): + The end of the range value. Must be the same type as + range_element_type. If not provided, it's interpreted as UNBOUNDED. + + name (Optional[str]): + Parameter name, used via ``@foo`` syntax. If None, the + parameter can only be addressed via position (``?``). + """ + + @classmethod + def _parse_range_element_type(self, range_element_type): + if isinstance(range_element_type, str): + if range_element_type not in _RANGE_ELEMENT_TYPE_STR: + raise ValueError( + "If given as a string, range_element_type must be one of " + f"'TIMESTAMP', 'DATE', or 'DATETIME'. Got {range_element_type}." + ) + return RangeQueryParameterType(range_element_type) + elif isinstance(range_element_type, RangeQueryParameterType): + if range_element_type.type_._type not in _RANGE_ELEMENT_TYPE_STR: + raise ValueError( + "If given as a RangeQueryParameterType object, " + "range_element_type must be one of 'TIMESTAMP', 'DATE', " + "or 'DATETIME' type." + ) + return range_element_type + else: + raise ValueError( + "range_element_type must be a string or " + "RangeQueryParameterType object, of 'TIMESTAMP', 'DATE', " + "or 'DATETIME' type. Got " + f"{type(range_element_type)}:{range_element_type}" + ) + + @classmethod + def _serialize_range_element_value(self, value, type_): + if value is None or isinstance(value, str): + return value + else: + converter = _SCALAR_VALUE_TO_JSON_PARAM.get(type_) + if converter is not None: + return converter(value) # type: ignore + else: + raise ValueError( + f"Cannot convert range element value from type {type_}, " + "must be one of the strings 'TIMESTAMP', 'DATE' " + "'DATETIME' or a RangeQueryParameterType object." + ) + + def __init__( + self, + range_element_type, + start=None, + end=None, + name=None, + ): + self.name = name + self.range_element_type = self._parse_range_element_type(range_element_type) + print(self.range_element_type.type_._type) + self.start = start + self.end = end + + @classmethod + def positional( + cls, range_element_type, start=None, end=None + ) -> "RangeQueryParameter": + """Factory for positional parameters. + + Args: + range_element_type (Union[str, RangeQueryParameterType]): + The type of range elements. It must be one of `'TIMESTAMP'`, + `'DATE'`, or `'DATETIME'`. + + start (Optional[Union[ScalarQueryParameter, str]]): + The start of the range value. Must be the same type as + range_element_type. If not provided, it's interpreted as + UNBOUNDED. + + end (Optional[Union[ScalarQueryParameter, str]]): + The end of the range value. Must be the same type as + range_element_type. If not provided, it's interpreted as + UNBOUNDED. + + Returns: + google.cloud.bigquery.query.RangeQueryParameter: Instance without + name. + """ + return cls(range_element_type, start, end) + + @classmethod + def from_api_repr(cls, resource: dict) -> "RangeQueryParameter": + """Factory: construct parameter from JSON resource. + + Args: + resource (Dict): JSON mapping of parameter + + Returns: + google.cloud.bigquery.query.RangeQueryParameter: Instance + """ + name = resource.get("name") + range_element_type = ( + resource.get("parameterType", {}).get("rangeElementType", {}).get("type") + ) + range_value = resource.get("parameterValue", {}).get("rangeValue", {}) + start = range_value.get("start", {}).get("value") + end = range_value.get("end", {}).get("value") + + return cls(range_element_type, start=start, end=end, name=name) + + def to_api_repr(self) -> dict: + """Construct JSON API representation for the parameter. + + Returns: + Dict: JSON mapping + """ + range_element_type = self.range_element_type.to_api_repr() + type_ = self.range_element_type.type_._type + start = self._serialize_range_element_value(self.start, type_) + end = self._serialize_range_element_value(self.end, type_) + resource = { + "parameterType": range_element_type, + "parameterValue": { + "rangeValue": { + "start": {"value": start}, + "end": {"value": end}, + }, + }, + } + + # distinguish between name not provided vs. name being empty string + if self.name is not None: + resource["name"] = self.name + + return resource + + def _key(self): + """A tuple key that uniquely describes this field. + + Used to compute this instance's hashcode and evaluate equality. + + Returns: + Tuple: The contents of this + :class:`~google.cloud.bigquery.query.RangeQueryParameter`. + """ + + range_element_type = self.range_element_type.to_api_repr() + return (self.name, range_element_type, self.start, self.end) + + def __eq__(self, other): + if not isinstance(other, RangeQueryParameter): + return NotImplemented + return self._key() == other._key() + + def __ne__(self, other): + return not self == other + + def __repr__(self): + return "RangeQueryParameter{}".format(self._key()) + + class SqlParameterScalarTypes: """Supported scalar SQL query parameter types as type objects.""" diff --git a/tests/system/test_query.py b/tests/system/test_query.py index 82be40693..0494272d9 100644 --- a/tests/system/test_query.py +++ b/tests/system/test_query.py @@ -26,6 +26,7 @@ from google.cloud.bigquery.query import ScalarQueryParameterType from google.cloud.bigquery.query import StructQueryParameter from google.cloud.bigquery.query import StructQueryParameterType +from google.cloud.bigquery.query import RangeQueryParameter @pytest.fixture(params=["INSERT", "QUERY"]) @@ -422,6 +423,38 @@ def test_query_statistics(bigquery_client, query_api_method): ) ], ), + ( + "SELECT @range_date", + "[2016-12-05, UNBOUNDED)", + [ + RangeQueryParameter( + name="range_date", + range_element_type="DATE", + start=datetime.date(2016, 12, 5), + ) + ], + ), + ( + "SELECT @range_datetime", + "[2016-12-05T00:00:00, UNBOUNDED)", + [ + RangeQueryParameter( + name="range_datetime", + range_element_type="DATETIME", + start=datetime.datetime(2016, 12, 5), + ) + ], + ), + ( + "SELECT @range_unbounded", + "[UNBOUNDED, UNBOUNDED)", + [ + RangeQueryParameter( + name="range_unbounded", + range_element_type="DATETIME", + ) + ], + ), ), ) def test_query_parameters( diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index 1704abac7..f511bf28d 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -376,6 +376,100 @@ def test_repr_all_optional_attrs(self): self.assertEqual(repr(param_type), expected) +class Test_RangeQueryParameterType(unittest.TestCase): + @staticmethod + def _get_target_class(): + from google.cloud.bigquery.query import RangeQueryParameterType + + return RangeQueryParameterType + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + def test_ctor_str(self): + param_type = self._make_one("DATE", name="foo", description="bar") + self.assertEqual(param_type.type_._type, "DATE") + self.assertEqual(param_type.name, "foo") + self.assertEqual(param_type.description, "bar") + + def test_ctor_type(self): + from google.cloud.bigquery import ScalarQueryParameterType + + scalar_type = ScalarQueryParameterType("DATE") + param_type = self._make_one(scalar_type, name="foo", description="bar") + self.assertEqual(param_type.type_._type, "DATE") + self.assertEqual(param_type.name, "foo") + self.assertEqual(param_type.description, "bar") + + def test_ctor_unsupported_type_str(self): + with self.assertRaises(ValueError): + self._make_one("TIME") + + def test_ctor_unsupported_type_type(self): + from google.cloud.bigquery import ScalarQueryParameterType + + scalar_type = ScalarQueryParameterType("TIME") + with self.assertRaises(ValueError): + self._make_one(scalar_type) + + def test_ctor_wrong_type(self): + with self.assertRaises(ValueError): + self._make_one(None) + + def test_from_api_repr(self): + RESOURCE = { + "type": "RANGE", + "rangeElementType": {"type": "DATE"}, + } + + klass = self._get_target_class() + result = klass.from_api_repr(RESOURCE) + self.assertEqual(result.type_._type, "DATE") + self.assertIsNone(result.name) + self.assertIsNone(result.description) + + def test_to_api_repr(self): + EXPECTED = { + "type": "RANGE", + "rangeElementType": {"type": "DATE"}, + } + param_type = self._make_one("DATE", name="foo", description="bar") + result = param_type.to_api_repr() + self.assertEqual(result, EXPECTED) + + def test__repr__(self): + param_type = self._make_one("DATE", name="foo", description="bar") + param_repr = "RangeQueryParameterType(ScalarQueryParameterType('DATE'), name='foo', description='bar')" + self.assertEqual(repr(param_type), param_repr) + + def test__eq__(self): + param_type1 = self._make_one("DATE", name="foo", description="bar") + self.assertEqual(param_type1, param_type1) + self.assertNotEqual(param_type1, object()) + + alias = self._make_one("DATE", name="foo", description="bar") + self.assertIsNot(param_type1, alias) + self.assertEqual(param_type1, alias) + + wrong_type = self._make_one("DATETIME", name="foo", description="bar") + self.assertNotEqual(param_type1, wrong_type) + + wrong_name = self._make_one("DATETIME", name="foo2", description="bar") + self.assertNotEqual(param_type1, wrong_name) + + wrong_description = self._make_one("DATETIME", name="foo", description="bar2") + self.assertNotEqual(param_type1, wrong_description) + + def test_with_name(self): + param_type1 = self._make_one("DATE", name="foo", description="bar") + param_type2 = param_type1.with_name("foo2") + + self.assertIsNot(param_type1, param_type2) + self.assertEqual(param_type2.type_._type, "DATE") + self.assertEqual(param_type2.name, "foo2") + self.assertEqual(param_type2.description, "bar") + + class Test__AbstractQueryParameter(unittest.TestCase): @staticmethod def _get_target_class(): @@ -663,6 +757,460 @@ def test___repr__(self): self.assertEqual(repr(field1), expected) +class Test_RangeQueryParameter(unittest.TestCase): + @staticmethod + def _get_target_class(): + from google.cloud.bigquery.query import RangeQueryParameter + + return RangeQueryParameter + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + def test_ctor(self): + from google.cloud.bigquery.query import RangeQueryParameterType + + range_element_type = RangeQueryParameterType(type_="DATE") + param = self._make_one( + range_element_type="DATE", start="2016-08-11", name="foo" + ) + self.assertEqual(param.name, "foo") + self.assertEqual(param.range_element_type, range_element_type) + self.assertEqual(param.start, "2016-08-11") + self.assertIs(param.end, None) + + def test_ctor_w_datetime_query_parameter_type_str(self): + from google.cloud.bigquery.query import RangeQueryParameterType + + range_element_type = RangeQueryParameterType(type_="DATETIME") + start_datetime = datetime.datetime(year=2020, month=12, day=31, hour=12) + end_datetime = datetime.datetime(year=2021, month=12, day=31, hour=12) + param = self._make_one( + range_element_type="DATETIME", + start=start_datetime, + end=end_datetime, + name="foo", + ) + self.assertEqual(param.range_element_type, range_element_type) + self.assertEqual(param.start, start_datetime) + self.assertEqual(param.end, end_datetime) + self.assertEqual(param.name, "foo") + + def test_ctor_w_datetime_query_parameter_type_type(self): + from google.cloud.bigquery.query import RangeQueryParameterType + + range_element_type = RangeQueryParameterType(type_="DATETIME") + param = self._make_one(range_element_type=range_element_type) + self.assertEqual(param.range_element_type, range_element_type) + self.assertEqual(param.start, None) + self.assertEqual(param.end, None) + self.assertEqual(param.name, None) + + def test_ctor_w_timestamp_query_parameter_typ_str(self): + from google.cloud.bigquery.query import RangeQueryParameterType + + range_element_type = RangeQueryParameterType(type_="TIMESTAMP") + start_datetime = datetime.datetime(year=2020, month=12, day=31, hour=12) + end_datetime = datetime.datetime(year=2021, month=12, day=31, hour=12) + param = self._make_one( + range_element_type="TIMESTAMP", + start=start_datetime, + end=end_datetime, + name="foo", + ) + self.assertEqual(param.range_element_type, range_element_type) + self.assertEqual(param.start, start_datetime) + self.assertEqual(param.end, end_datetime) + self.assertEqual(param.name, "foo") + + def test_ctor_w_timestamp_query_parameter_type_type(self): + from google.cloud.bigquery.query import RangeQueryParameterType + + range_element_type = RangeQueryParameterType(type_="TIMESTAMP") + param = self._make_one(range_element_type=range_element_type) + self.assertEqual(param.range_element_type, range_element_type) + self.assertEqual(param.start, None) + self.assertEqual(param.end, None) + self.assertEqual(param.name, None) + + def test_ctor_w_date_query_parameter_type_str(self): + from google.cloud.bigquery.query import RangeQueryParameterType + + range_element_type = RangeQueryParameterType(type_="DATE") + start_date = datetime.date(year=2020, month=12, day=31) + end_date = datetime.date(year=2021, month=12, day=31) + param = self._make_one( + range_element_type="DATE", + start=start_date, + end=end_date, + name="foo", + ) + self.assertEqual(param.range_element_type, range_element_type) + self.assertEqual(param.start, start_date) + self.assertEqual(param.end, end_date) + self.assertEqual(param.name, "foo") + + def test_ctor_w_date_query_parameter_type_type(self): + from google.cloud.bigquery.query import RangeQueryParameterType + + range_element_type = RangeQueryParameterType(type_="DATE") + param = self._make_one(range_element_type=range_element_type) + self.assertEqual(param.range_element_type, range_element_type) + self.assertEqual(param.start, None) + self.assertEqual(param.end, None) + self.assertEqual(param.name, None) + + def test_ctor_w_name_empty_str(self): + from google.cloud.bigquery.query import RangeQueryParameterType + + range_element_type = RangeQueryParameterType(type_="DATE") + param = self._make_one( + range_element_type="DATE", + name="", + ) + self.assertEqual(param.range_element_type, range_element_type) + self.assertIs(param.start, None) + self.assertIs(param.end, None) + self.assertEqual(param.name, "") + + def test_ctor_wo_value(self): + from google.cloud.bigquery.query import RangeQueryParameterType + + range_element_type = RangeQueryParameterType(type_="DATETIME") + param = self._make_one(range_element_type="DATETIME", name="foo") + self.assertEqual(param.range_element_type, range_element_type) + self.assertIs(param.start, None) + self.assertIs(param.end, None) + self.assertEqual(param.name, "foo") + + def test_ctor_w_unsupported_query_parameter_type_str(self): + with self.assertRaises(ValueError): + self._make_one(range_element_type="TIME", name="foo") + + def test_ctor_w_unsupported_query_parameter_type_type(self): + from google.cloud.bigquery.query import RangeQueryParameterType + + range_element_type = RangeQueryParameterType(type_="DATE") + range_element_type.type_._type = "TIME" + with self.assertRaises(ValueError): + self._make_one(range_element_type=range_element_type, name="foo") + + def test_ctor_w_unsupported_query_parameter_type_input(self): + with self.assertRaises(ValueError): + self._make_one(range_element_type=None, name="foo") + + def test_positional(self): + from google.cloud.bigquery.query import RangeQueryParameterType + + range_element_type = RangeQueryParameterType(type_="DATE") + klass = self._get_target_class() + param = klass.positional( + range_element_type="DATE", start="2016-08-11", end="2016-08-12" + ) + self.assertIs(param.name, None) + self.assertEqual(param.range_element_type, range_element_type) + self.assertEqual(param.start, "2016-08-11") + self.assertEqual(param.end, "2016-08-12") + + def test_from_api_repr_w_name(self): + from google.cloud.bigquery.query import RangeQueryParameterType + + RESOURCE = { + "name": "foo", + "parameterType": { + "type": "RANGE", + "rangeElementType": { + "type": "DATE", + }, + }, + "parameterValue": { + "rangeValue": {"start": {"value": None}, "end": {"value": "2020-12-31"}} + }, + } + klass = self._get_target_class() + param = klass.from_api_repr(RESOURCE) + range_element_type = RangeQueryParameterType(type_="DATE") + self.assertEqual(param.name, "foo") + self.assertEqual(param.range_element_type, range_element_type) + self.assertEqual(param.start, None) + self.assertEqual(param.end, "2020-12-31") + + def test_from_api_repr_wo_name(self): + from google.cloud.bigquery.query import RangeQueryParameterType + + RESOURCE = { + "parameterType": { + "type": "RANGE", + "rangeElementType": { + "type": "DATE", + }, + }, + "parameterValue": { + "rangeValue": {"start": {"value": None}, "end": {"value": "2020-12-31"}} + }, + } + klass = self._get_target_class() + param = klass.from_api_repr(RESOURCE) + range_element_type = RangeQueryParameterType(type_="DATE") + self.assertEqual(param.name, None) + self.assertEqual(param.range_element_type, range_element_type) + self.assertEqual(param.start, None) + self.assertEqual(param.end, "2020-12-31") + + def test_from_api_repr_wo_value(self): + # Back-end may not send back values for None params. See #9027 + from google.cloud.bigquery.query import RangeQueryParameterType + + RESOURCE = { + "parameterType": { + "type": "RANGE", + "rangeElementType": { + "type": "DATE", + }, + }, + } + range_element_type = RangeQueryParameterType(type_="DATE") + klass = self._get_target_class() + param = klass.from_api_repr(RESOURCE) + self.assertIs(param.name, None) + self.assertEqual(param.range_element_type, range_element_type) + self.assertIs(param.start, None) + self.assertIs(param.end, None) + + def test_to_api_repr_w_name(self): + EXPECTED = { + "name": "foo", + "parameterType": { + "type": "RANGE", + "rangeElementType": { + "type": "DATE", + }, + }, + "parameterValue": { + "rangeValue": {"start": {"value": None}, "end": {"value": "2016-08-11"}} + }, + } + param = self._make_one(range_element_type="DATE", end="2016-08-11", name="foo") + self.assertEqual(param.to_api_repr(), EXPECTED) + + def test_to_api_repr_wo_name(self): + EXPECTED = { + "parameterType": { + "type": "RANGE", + "rangeElementType": { + "type": "DATE", + }, + }, + "parameterValue": { + "rangeValue": {"start": {"value": None}, "end": {"value": "2016-08-11"}} + }, + } + klass = self._get_target_class() + param = klass.positional(range_element_type="DATE", end="2016-08-11") + self.assertEqual(param.to_api_repr(), EXPECTED) + + def test_to_api_repr_w_date_date(self): + today = datetime.date.today() + today_str = today.strftime("%Y-%m-%d") + EXPECTED = { + "name": "foo", + "parameterType": { + "type": "RANGE", + "rangeElementType": { + "type": "DATE", + }, + }, + "parameterValue": { + "rangeValue": {"start": {"value": None}, "end": {"value": today_str}} + }, + } + param = self._make_one(range_element_type="DATE", end=today, name="foo") + self.assertEqual(param.to_api_repr(), EXPECTED) + + def test_to_api_repr_w_datetime_str(self): + EXPECTED = { + "parameterType": { + "type": "RANGE", + "rangeElementType": { + "type": "DATETIME", + }, + }, + "parameterValue": { + "rangeValue": { + "start": {"value": None}, + "end": {"value": "2020-01-01T12:00:00.000000"}, + } + }, + } + klass = self._get_target_class() + end_datetime = datetime.datetime(year=2020, month=1, day=1, hour=12) + param = klass.positional(range_element_type="DATETIME", end=end_datetime) + self.assertEqual(param.to_api_repr(), EXPECTED) + + def test_to_api_repr_w_datetime_datetime(self): + from google.cloud.bigquery._helpers import _RFC3339_MICROS_NO_ZULU + + now = datetime.datetime.utcnow() + now_str = now.strftime(_RFC3339_MICROS_NO_ZULU) + EXPECTED = { + "parameterType": { + "type": "RANGE", + "rangeElementType": { + "type": "DATETIME", + }, + }, + "parameterValue": { + "rangeValue": {"start": {"value": None}, "end": {"value": now_str}} + }, + } + klass = self._get_target_class() + param = klass.positional(range_element_type="DATETIME", end=now) + self.assertEqual(param.to_api_repr(), EXPECTED) + + def test_to_api_repr_w_timestamp_str(self): + EXPECTED = { + "parameterType": { + "type": "RANGE", + "rangeElementType": { + "type": "TIMESTAMP", + }, + }, + "parameterValue": { + "rangeValue": { + "start": {"value": None}, + "end": {"value": "2020-01-01 12:00:00+00:00"}, + } + }, + } + klass = self._get_target_class() + end_timestamp = datetime.datetime(year=2020, month=1, day=1, hour=12) + param = klass.positional(range_element_type="TIMESTAMP", end=end_timestamp) + self.assertEqual(param.to_api_repr(), EXPECTED) + + def test_to_api_repr_w_timestamp_timestamp(self): + from google.cloud._helpers import UTC # type: ignore + + now = datetime.datetime.utcnow() + now = now.astimezone(UTC) + now_str = str(now) + EXPECTED = { + "parameterType": { + "type": "RANGE", + "rangeElementType": { + "type": "TIMESTAMP", + }, + }, + "parameterValue": { + "rangeValue": {"start": {"value": None}, "end": {"value": now_str}} + }, + } + klass = self._get_target_class() + param = klass.positional(range_element_type="TIMESTAMP", end=now) + self.assertEqual(param.to_api_repr(), EXPECTED) + + def test_to_api_repr_wo_values(self): + EXPECTED = { + "name": "foo", + "parameterType": { + "type": "RANGE", + "rangeElementType": { + "type": "DATE", + }, + }, + "parameterValue": { + "rangeValue": {"start": {"value": None}, "end": {"value": None}} + }, + } + param = self._make_one(range_element_type="DATE", name="foo") + self.assertEqual(param.to_api_repr(), EXPECTED) + + def test_to_api_repr_unsupported_value_type(self): + with self.assertRaisesRegex( + ValueError, "Cannot convert range element value from type" + ): + range_param = self._make_one( + range_element_type="DATE", start=datetime.date.today() + ) + range_param.range_element_type.type_._type = "LONG" + range_param.to_api_repr() + + def test___eq__(self): + param = self._make_one( + range_element_type="DATE", start="2016-08-11", name="foo" + ) + self.assertEqual(param, param) + self.assertNotEqual(param, object()) + alias = self._make_one( + range_element_type="DATE", start="2016-08-11", name="bar" + ) + self.assertNotEqual(param, alias) + wrong_type = self._make_one( + range_element_type="DATETIME", + start="2020-12-31 12:00:00.000000", + name="foo", + ) + self.assertNotEqual(param, wrong_type) + wrong_val = self._make_one( + range_element_type="DATE", start="2016-08-12", name="foo" + ) + self.assertNotEqual(param, wrong_val) + + def test___eq___wrong_type(self): + param = self._make_one( + range_element_type="DATE", start="2016-08-11", name="foo" + ) + other = object() + self.assertNotEqual(param, other) + self.assertEqual(param, mock.ANY) + + def test___eq___name_mismatch(self): + param = self._make_one( + range_element_type="DATE", start="2016-08-11", name="foo" + ) + other = self._make_one( + range_element_type="DATE", start="2016-08-11", name="bar" + ) + self.assertNotEqual(param, other) + + def test___eq___field_type_mismatch(self): + param = self._make_one(range_element_type="DATE") + other = self._make_one(range_element_type="DATETIME") + self.assertNotEqual(param, other) + + def test___eq___value_mismatch(self): + param = self._make_one(range_element_type="DATE", start="2016-08-11") + other = self._make_one(range_element_type="DATE", start="2016-08-12") + self.assertNotEqual(param, other) + + def test___eq___hit(self): + param = self._make_one(range_element_type="DATE", start="2016-08-12") + other = self._make_one(range_element_type="DATE", start="2016-08-12") + self.assertEqual(param, other) + + def test___ne___wrong_type(self): + param = self._make_one(range_element_type="DATE") + other = object() + self.assertNotEqual(param, other) + self.assertEqual(param, mock.ANY) + + def test___ne___same_value(self): + param1 = self._make_one(range_element_type="DATE") + param2 = self._make_one(range_element_type="DATE") + # unittest ``assertEqual`` uses ``==`` not ``!=``. + comparison_val = param1 != param2 + self.assertFalse(comparison_val) + + def test___ne___different_values(self): + param1 = self._make_one(range_element_type="DATE", start="2016-08-12") + param2 = self._make_one(range_element_type="DATE") + self.assertNotEqual(param1, param2) + + def test___repr__(self): + param1 = self._make_one(range_element_type="DATE", start="2016-08-12") + expected = "RangeQueryParameter(None, {'type': 'RANGE', 'rangeElementType': {'type': 'DATE'}}, '2016-08-12', None)" + self.assertEqual(repr(param1), expected) + + def _make_subparam(name, type_, value): from google.cloud.bigquery.query import ScalarQueryParameter From 53c2cbf98d2961f553747514de273bcd5c117f0e Mon Sep 17 00:00:00 2001 From: shollyman Date: Tue, 5 Mar 2024 09:47:31 -0800 Subject: [PATCH 02/10] fix: augment universe_domain handling (#1837) * fix: augment universe_domain handling This PR revisits the universe resolution for the BQ client, and handles new requirements like env-based specification and validation. * lint * skipif core too old * deps * add import * no-cover in test helper * lint * ignore google.auth typing * capitalization * change to raise in test code * reviewer feedback * var fix --------- Co-authored-by: Lingqing Gan --- google/cloud/bigquery/_helpers.py | 56 ++++++++++++++++++++++ google/cloud/bigquery/client.py | 21 ++++---- tests/unit/helpers.py | 14 ++++++ tests/unit/test__helpers.py | 80 ++++++++++++++++++++++++++++++- 4 files changed, 162 insertions(+), 9 deletions(-) diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py index 905d4aee1..ec4ac9970 100644 --- a/google/cloud/bigquery/_helpers.py +++ b/google/cloud/bigquery/_helpers.py @@ -30,6 +30,8 @@ from google.cloud._helpers import _RFC3339_MICROS from google.cloud._helpers import _RFC3339_NO_FRACTION from google.cloud._helpers import _to_bytes +from google.auth import credentials as ga_credentials # type: ignore +from google.api_core import client_options as client_options_lib _RFC3339_MICROS_NO_ZULU = "%Y-%m-%dT%H:%M:%S.%f" _TIMEONLY_WO_MICROS = "%H:%M:%S" @@ -55,9 +57,63 @@ _DEFAULT_HOST = "https://bigquery.googleapis.com" """Default host for JSON API.""" +_DEFAULT_HOST_TEMPLATE = "https://bigquery.{UNIVERSE_DOMAIN}" +""" Templatized endpoint format. """ + _DEFAULT_UNIVERSE = "googleapis.com" """Default universe for the JSON API.""" +_UNIVERSE_DOMAIN_ENV = "GOOGLE_CLOUD_UNIVERSE_DOMAIN" +"""Environment variable for setting universe domain.""" + + +def _get_client_universe( + client_options: Optional[Union[client_options_lib.ClientOptions, dict]] +) -> str: + """Retrieves the specified universe setting. + + Args: + client_options: specified client options. + Returns: + str: resolved universe setting. + + """ + if isinstance(client_options, dict): + client_options = client_options_lib.from_dict(client_options) + universe = _DEFAULT_UNIVERSE + if hasattr(client_options, "universe_domain"): + options_universe = getattr(client_options, "universe_domain") + if options_universe is not None and len(options_universe) > 0: + universe = options_universe + else: + env_universe = os.getenv(_UNIVERSE_DOMAIN_ENV) + if isinstance(env_universe, str) and len(env_universe) > 0: + universe = env_universe + return universe + + +def _validate_universe(client_universe: str, credentials: ga_credentials.Credentials): + """Validates that client provided universe and universe embedded in credentials match. + + Args: + client_universe (str): The universe domain configured via the client options. + credentials (ga_credentials.Credentials): The credentials being used in the client. + + Raises: + ValueError: when client_universe does not match the universe in credentials. + """ + if hasattr(credentials, "universe_domain"): + cred_universe = getattr(credentials, "universe_domain") + if isinstance(cred_universe, str): + if client_universe != cred_universe: + raise ValueError( + "The configured universe domain " + f"({client_universe}) does not match the universe domain " + f"found in the credentials ({cred_universe}). " + "If you haven't configured the universe domain explicitly, " + f"`{_DEFAULT_UNIVERSE}` is the default." + ) + def _get_bigquery_host(): return os.environ.get(BIGQUERY_EMULATOR_HOST, _DEFAULT_HOST) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index a871dc003..cb4daa897 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -78,7 +78,10 @@ from google.cloud.bigquery._helpers import _verify_job_config_type from google.cloud.bigquery._helpers import _get_bigquery_host from google.cloud.bigquery._helpers import _DEFAULT_HOST +from google.cloud.bigquery._helpers import _DEFAULT_HOST_TEMPLATE from google.cloud.bigquery._helpers import _DEFAULT_UNIVERSE +from google.cloud.bigquery._helpers import _validate_universe +from google.cloud.bigquery._helpers import _get_client_universe from google.cloud.bigquery._job_helpers import make_job_id as _make_job_id from google.cloud.bigquery.dataset import Dataset from google.cloud.bigquery.dataset import DatasetListItem @@ -245,6 +248,7 @@ def __init__( kw_args = {"client_info": client_info} bq_host = _get_bigquery_host() kw_args["api_endpoint"] = bq_host if bq_host != _DEFAULT_HOST else None + client_universe = None if client_options: if isinstance(client_options, dict): client_options = google.api_core.client_options.from_dict( @@ -253,14 +257,15 @@ def __init__( if client_options.api_endpoint: api_endpoint = client_options.api_endpoint kw_args["api_endpoint"] = api_endpoint - elif ( - hasattr(client_options, "universe_domain") - and client_options.universe_domain - and client_options.universe_domain is not _DEFAULT_UNIVERSE - ): - kw_args["api_endpoint"] = _DEFAULT_HOST.replace( - _DEFAULT_UNIVERSE, client_options.universe_domain - ) + else: + client_universe = _get_client_universe(client_options) + if client_universe != _DEFAULT_UNIVERSE: + kw_args["api_endpoint"] = _DEFAULT_HOST_TEMPLATE.replace( + "{UNIVERSE_DOMAIN}", client_universe + ) + # Ensure credentials and universe are not in conflict. + if hasattr(self, "_credentials") and client_universe is not None: + _validate_universe(client_universe, self._credentials) self._connection = Connection(self, **kw_args) self._location = location diff --git a/tests/unit/helpers.py b/tests/unit/helpers.py index 67aeaca35..bc92c0df6 100644 --- a/tests/unit/helpers.py +++ b/tests/unit/helpers.py @@ -43,6 +43,20 @@ def make_client(project="PROJECT", **kw): return google.cloud.bigquery.client.Client(project, credentials, **kw) +def make_creds(creds_universe: None): + from google.auth import credentials + + class TestingCreds(credentials.Credentials): + def refresh(self, request): # pragma: NO COVER + raise NotImplementedError + + @property + def universe_domain(self): + return creds_universe + + return TestingCreds() + + def make_dataset_reference_string(project, ds_id): return f"{project}.{ds_id}" diff --git a/tests/unit/test__helpers.py b/tests/unit/test__helpers.py index 87ab46669..019d2e7bd 100644 --- a/tests/unit/test__helpers.py +++ b/tests/unit/test__helpers.py @@ -17,8 +17,86 @@ import decimal import json import unittest - +import os import mock +import pytest +import packaging +import google.api_core + + +@pytest.mark.skipif( + packaging.version.parse(getattr(google.api_core, "__version__", "0.0.0")) + < packaging.version.Version("2.15.0"), + reason="universe_domain not supported with google-api-core < 2.15.0", +) +class Test_get_client_universe(unittest.TestCase): + def test_with_none(self): + from google.cloud.bigquery._helpers import _get_client_universe + + self.assertEqual("googleapis.com", _get_client_universe(None)) + + def test_with_dict(self): + from google.cloud.bigquery._helpers import _get_client_universe + + options = {"universe_domain": "foo.com"} + self.assertEqual("foo.com", _get_client_universe(options)) + + def test_with_dict_empty(self): + from google.cloud.bigquery._helpers import _get_client_universe + + options = {"universe_domain": ""} + self.assertEqual("googleapis.com", _get_client_universe(options)) + + def test_with_client_options(self): + from google.cloud.bigquery._helpers import _get_client_universe + from google.api_core import client_options + + options = client_options.from_dict({"universe_domain": "foo.com"}) + self.assertEqual("foo.com", _get_client_universe(options)) + + @mock.patch.dict(os.environ, {"GOOGLE_CLOUD_UNIVERSE_DOMAIN": "foo.com"}) + def test_with_environ(self): + from google.cloud.bigquery._helpers import _get_client_universe + + self.assertEqual("foo.com", _get_client_universe(None)) + + @mock.patch.dict(os.environ, {"GOOGLE_CLOUD_UNIVERSE_DOMAIN": ""}) + def test_with_environ_empty(self): + from google.cloud.bigquery._helpers import _get_client_universe + + self.assertEqual("googleapis.com", _get_client_universe(None)) + + +class Test_validate_universe(unittest.TestCase): + def test_with_none(self): + from google.cloud.bigquery._helpers import _validate_universe + + # should not raise + _validate_universe("googleapis.com", None) + + def test_with_no_universe_creds(self): + from google.cloud.bigquery._helpers import _validate_universe + from .helpers import make_creds + + creds = make_creds(None) + # should not raise + _validate_universe("googleapis.com", creds) + + def test_with_matched_universe_creds(self): + from google.cloud.bigquery._helpers import _validate_universe + from .helpers import make_creds + + creds = make_creds("googleapis.com") + # should not raise + _validate_universe("googleapis.com", creds) + + def test_with_mismatched_universe_creds(self): + from google.cloud.bigquery._helpers import _validate_universe + from .helpers import make_creds + + creds = make_creds("foo.com") + with self.assertRaises(ValueError): + _validate_universe("googleapis.com", creds) class Test_not_null(unittest.TestCase): From 86a45c989836b34dca456bac014352e55d6f86c0 Mon Sep 17 00:00:00 2001 From: Lingqing Gan Date: Wed, 6 Mar 2024 17:04:06 -0800 Subject: [PATCH 03/10] feat: support range sql (#1807) * feat: support range sql * add unit tests * add system test * lint and remove debug code * lint and remove debug code * remove added blank line * add comment for legacy type --- google/cloud/bigquery/enums.py | 2 ++ google/cloud/bigquery/standard_sql.py | 36 ++++++++++++++++++- tests/system/test_client.py | 38 ++++++++++++++++++++ tests/unit/test_standard_sql_types.py | 52 +++++++++++++++++++++++++++ 4 files changed, 127 insertions(+), 1 deletion(-) diff --git a/google/cloud/bigquery/enums.py b/google/cloud/bigquery/enums.py index 553853630..d75037ad1 100644 --- a/google/cloud/bigquery/enums.py +++ b/google/cloud/bigquery/enums.py @@ -254,6 +254,7 @@ def _generate_next_value_(name, start, count, last_values): JSON = enum.auto() ARRAY = enum.auto() STRUCT = enum.auto() + RANGE = enum.auto() class EntityTypes(str, enum.Enum): @@ -292,6 +293,7 @@ class SqlTypeNames(str, enum.Enum): TIME = "TIME" DATETIME = "DATETIME" INTERVAL = "INTERVAL" # NOTE: not available in legacy types + RANGE = "RANGE" # NOTE: not available in legacy types class WriteDisposition(object): diff --git a/google/cloud/bigquery/standard_sql.py b/google/cloud/bigquery/standard_sql.py index e0f22b2de..68332eb80 100644 --- a/google/cloud/bigquery/standard_sql.py +++ b/google/cloud/bigquery/standard_sql.py @@ -43,6 +43,7 @@ class StandardSqlDataType: ] } } + RANGE: {type_kind="RANGE", range_element_type="DATETIME"} Args: type_kind: @@ -52,6 +53,8 @@ class StandardSqlDataType: The type of the array's elements, if type_kind is ARRAY. struct_type: The fields of this struct, in order, if type_kind is STRUCT. + range_element_type: + The type of the range's elements, if type_kind is RANGE. """ def __init__( @@ -61,12 +64,14 @@ def __init__( ] = StandardSqlTypeNames.TYPE_KIND_UNSPECIFIED, array_element_type: Optional["StandardSqlDataType"] = None, struct_type: Optional["StandardSqlStructType"] = None, + range_element_type: Optional["StandardSqlDataType"] = None, ): self._properties: Dict[str, Any] = {} self.type_kind = type_kind self.array_element_type = array_element_type self.struct_type = struct_type + self.range_element_type = range_element_type @property def type_kind(self) -> Optional[StandardSqlTypeNames]: @@ -127,6 +132,28 @@ def struct_type(self, value: Optional["StandardSqlStructType"]): else: self._properties["structType"] = struct_type + @property + def range_element_type(self) -> Optional["StandardSqlDataType"]: + """The type of the range's elements, if type_kind = "RANGE". Must be + one of DATETIME, DATE, or TIMESTAMP.""" + range_element_info = self._properties.get("rangeElementType") + + if range_element_info is None: + return None + + result = StandardSqlDataType() + result._properties = range_element_info # We do not use a copy on purpose. + return result + + @range_element_type.setter + def range_element_type(self, value: Optional["StandardSqlDataType"]): + range_element_type = None if value is None else value.to_api_repr() + + if range_element_type is None: + self._properties.pop("rangeElementType", None) + else: + self._properties["rangeElementType"] = range_element_type + def to_api_repr(self) -> Dict[str, Any]: """Construct the API resource representation of this SQL data type.""" return copy.deepcopy(self._properties) @@ -155,7 +182,13 @@ def from_api_repr(cls, resource: Dict[str, Any]): if struct_info: struct_type = StandardSqlStructType.from_api_repr(struct_info) - return cls(type_kind, array_element_type, struct_type) + range_element_type = None + if type_kind == StandardSqlTypeNames.RANGE: + range_element_info = resource.get("rangeElementType") + if range_element_info: + range_element_type = cls.from_api_repr(range_element_info) + + return cls(type_kind, array_element_type, struct_type, range_element_type) def __eq__(self, other): if not isinstance(other, StandardSqlDataType): @@ -165,6 +198,7 @@ def __eq__(self, other): self.type_kind == other.type_kind and self.array_element_type == other.array_element_type and self.struct_type == other.struct_type + and self.range_element_type == other.range_element_type ) def __str__(self): diff --git a/tests/system/test_client.py b/tests/system/test_client.py index 74c152cf2..04740de8a 100644 --- a/tests/system/test_client.py +++ b/tests/system/test_client.py @@ -2193,6 +2193,44 @@ def test_create_routine(self): assert len(rows) == 1 assert rows[0].max_value == 100.0 + def test_create_routine_with_range(self): + routine_name = "routine_range" + dataset = self.temp_dataset(_make_dataset_id("routine_range")) + + routine = bigquery.Routine( + dataset.routine(routine_name), + type_="SCALAR_FUNCTION", + language="SQL", + body="RANGE_START(x)", + arguments=[ + bigquery.RoutineArgument( + name="x", + data_type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.RANGE, + range_element_type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.DATE + ), + ), + ) + ], + return_type=bigquery.StandardSqlDataType( + type_kind=bigquery.StandardSqlTypeNames.DATE + ), + ) + + query_string = ( + "SELECT `{}`(RANGE '[2016-08-12, UNBOUNDED)') as range_start;".format( + str(routine.reference) + ) + ) + + routine = helpers.retry_403(Config.CLIENT.create_routine)(routine) + query_job = helpers.retry_403(Config.CLIENT.query)(query_string) + rows = list(query_job.result()) + + assert len(rows) == 1 + assert rows[0].range_start == datetime.date(2016, 8, 12) + def test_create_tvf_routine(self): from google.cloud.bigquery import ( Routine, diff --git a/tests/unit/test_standard_sql_types.py b/tests/unit/test_standard_sql_types.py index 0ba0e0cfd..3ed912b5a 100644 --- a/tests/unit/test_standard_sql_types.py +++ b/tests/unit/test_standard_sql_types.py @@ -129,6 +129,28 @@ def test_to_api_repr_struct_type_w_field_types(self): } assert result == expected + def test_to_api_repr_range_type_element_type_missing(self): + instance = self._make_one( + bq.StandardSqlTypeNames.RANGE, range_element_type=None + ) + + result = instance.to_api_repr() + + assert result == {"typeKind": "RANGE"} + + def test_to_api_repr_range_type_w_element_type(self): + range_element_type = self._make_one(type_kind=bq.StandardSqlTypeNames.DATE) + instance = self._make_one( + bq.StandardSqlTypeNames.RANGE, range_element_type=range_element_type + ) + + result = instance.to_api_repr() + + assert result == { + "typeKind": "RANGE", + "rangeElementType": {"typeKind": "DATE"}, + } + def test_from_api_repr_empty_resource(self): klass = self._get_target_class() result = klass.from_api_repr(resource={}) @@ -276,6 +298,31 @@ def test_from_api_repr_struct_type_incomplete_field_info(self): ) assert result == expected + def test_from_api_repr_range_type_full(self): + klass = self._get_target_class() + resource = {"typeKind": "RANGE", "rangeElementType": {"typeKind": "DATE"}} + + result = klass.from_api_repr(resource=resource) + + expected = klass( + type_kind=bq.StandardSqlTypeNames.RANGE, + range_element_type=klass(type_kind=bq.StandardSqlTypeNames.DATE), + ) + assert result == expected + + def test_from_api_repr_range_type_missing_element_type(self): + klass = self._get_target_class() + resource = {"typeKind": "RANGE"} + + result = klass.from_api_repr(resource=resource) + + expected = klass( + type_kind=bq.StandardSqlTypeNames.RANGE, + range_element_type=None, + struct_type=None, + ) + assert result == expected + def test__eq__another_type(self): instance = self._make_one() @@ -321,6 +368,11 @@ def test__eq__similar_instance(self): bq.StandardSqlStructType(fields=[bq.StandardSqlField(name="foo")]), bq.StandardSqlStructType(fields=[bq.StandardSqlField(name="bar")]), ), + ( + "range_element_type", + bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.DATE), + bq.StandardSqlDataType(type_kind=bq.StandardSqlTypeNames.DATETIME), + ), ), ) def test__eq__attribute_differs(self, attr_name, value, value2): From b8189929b6008f7780214822062f8ed05d8d2a01 Mon Sep 17 00:00:00 2001 From: shollyman Date: Wed, 6 Mar 2024 17:42:18 -0800 Subject: [PATCH 04/10] fix: supplementary fix to env-based universe resolution (#1844) * fix: supplementary fix to env-based universe resolution There's a corner case where conversion from dict to a ClientOptions will return a universe_domain value as None that wasn't covered by initial testing. This updates the resolution code and adds tests to exercise the new path. * formatting --------- Co-authored-by: Lingqing Gan --- google/cloud/bigquery/_helpers.py | 11 +++++++---- tests/unit/test__helpers.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py index ec4ac9970..7198b60c2 100644 --- a/google/cloud/bigquery/_helpers.py +++ b/google/cloud/bigquery/_helpers.py @@ -81,10 +81,13 @@ def _get_client_universe( if isinstance(client_options, dict): client_options = client_options_lib.from_dict(client_options) universe = _DEFAULT_UNIVERSE - if hasattr(client_options, "universe_domain"): - options_universe = getattr(client_options, "universe_domain") - if options_universe is not None and len(options_universe) > 0: - universe = options_universe + options_universe = getattr(client_options, "universe_domain", None) + if ( + options_universe + and isinstance(options_universe, str) + and len(options_universe) > 0 + ): + universe = options_universe else: env_universe = os.getenv(_UNIVERSE_DOMAIN_ENV) if isinstance(env_universe, str) and len(env_universe) > 0: diff --git a/tests/unit/test__helpers.py b/tests/unit/test__helpers.py index 019d2e7bd..7e8d815d2 100644 --- a/tests/unit/test__helpers.py +++ b/tests/unit/test__helpers.py @@ -60,6 +60,21 @@ def test_with_environ(self): self.assertEqual("foo.com", _get_client_universe(None)) + @mock.patch.dict(os.environ, {"GOOGLE_CLOUD_UNIVERSE_DOMAIN": "foo.com"}) + def test_with_environ_and_dict(self): + from google.cloud.bigquery._helpers import _get_client_universe + + options = ({"credentials_file": "file.json"},) + self.assertEqual("foo.com", _get_client_universe(options)) + + @mock.patch.dict(os.environ, {"GOOGLE_CLOUD_UNIVERSE_DOMAIN": "foo.com"}) + def test_with_environ_and_empty_options(self): + from google.cloud.bigquery._helpers import _get_client_universe + from google.api_core import client_options + + options = client_options.from_dict({}) + self.assertEqual("foo.com", _get_client_universe(options)) + @mock.patch.dict(os.environ, {"GOOGLE_CLOUD_UNIVERSE_DOMAIN": ""}) def test_with_environ_empty(self): from google.cloud.bigquery._helpers import _get_client_universe From 08762fbf3289622c2b948da919ae49567d0378d5 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Thu, 7 Mar 2024 15:10:32 -0500 Subject: [PATCH 05/10] chore: use mock from unittest (#1823) * chore: use mock from unittest * correct user_credentials_test.py * add try except for Python 3.7 * fixes linting * adjustments to testing suite to account for dependencies * updates to mypy_samples.py * linting for noxfile.py --------- Co-authored-by: Chalmer Lowe --- noxfile.py | 8 ++++---- samples/desktopapp/user_credentials_test.py | 11 +++++++++-- samples/tests/conftest.py | 2 +- testing/constraints-3.7.txt | 13 +++++++++---- tests/unit/conftest.py | 3 ++- tests/unit/helpers.py | 7 ++++--- tests/unit/job/test_base.py | 2 +- tests/unit/job/test_copy.py | 2 +- tests/unit/job/test_extract.py | 2 +- tests/unit/job/test_load.py | 3 +-- tests/unit/job/test_query.py | 16 ++++++++-------- tests/unit/job/test_query_pandas.py | 2 +- tests/unit/test__helpers.py | 5 +++-- tests/unit/test__http.py | 2 +- tests/unit/test__pandas_helpers.py | 7 +++---- tests/unit/test__versions_helpers.py | 4 ++-- tests/unit/test_client.py | 13 ++++++------- tests/unit/test_create_dataset.py | 3 ++- tests/unit/test_dataset.py | 2 +- tests/unit/test_dbapi_connection.py | 3 +-- tests/unit/test_dbapi_cursor.py | 2 +- tests/unit/test_encryption_configuration.py | 2 +- tests/unit/test_job_retry.py | 2 +- tests/unit/test_list_datasets.py | 3 ++- tests/unit/test_list_jobs.py | 2 +- tests/unit/test_list_projects.py | 3 ++- tests/unit/test_magics.py | 6 +++--- tests/unit/test_opentelemetry_tracing.py | 3 +-- tests/unit/test_query.py | 3 +-- tests/unit/test_retry.py | 2 +- tests/unit/test_schema.py | 2 +- tests/unit/test_table.py | 2 +- 32 files changed, 77 insertions(+), 65 deletions(-) diff --git a/noxfile.py b/noxfile.py index ae022232e..c31d098b8 100644 --- a/noxfile.py +++ b/noxfile.py @@ -72,7 +72,6 @@ def default(session, install_extras=True): # Install all test dependencies, then install local packages in-place. session.install( - "mock", "pytest", "google-cloud-testutils", "pytest-cov", @@ -89,6 +88,8 @@ def default(session, install_extras=True): install_target = "." session.install("-e", install_target, "-c", constraints_path) + session.run("python", "-m", "pip", "freeze") + # Run py.test against the unit tests. session.run( "py.test", @@ -176,7 +177,7 @@ def system(session): # Install all test dependencies, then install local packages in place. session.install( - "mock", "pytest", "psutil", "google-cloud-testutils", "-c", constraints_path + "pytest", "psutil", "google-cloud-testutils", "-c", constraints_path ) if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "") == "true": # mTLS test requires pyopenssl and latest google-cloud-storage @@ -249,7 +250,7 @@ def snippets(session): ) # Install all test dependencies, then install local packages in place. - session.install("mock", "pytest", "google-cloud-testutils", "-c", constraints_path) + session.install("pytest", "google-cloud-testutils", "-c", constraints_path) session.install("google-cloud-storage", "-c", constraints_path) session.install("grpcio", "-c", constraints_path) @@ -336,7 +337,6 @@ def prerelease_deps(session): "google-cloud-datacatalog", "google-cloud-storage", "google-cloud-testutils", - "mock", "psutil", "pytest", "pytest-cov", diff --git a/samples/desktopapp/user_credentials_test.py b/samples/desktopapp/user_credentials_test.py index baa9e33f1..252b843c4 100644 --- a/samples/desktopapp/user_credentials_test.py +++ b/samples/desktopapp/user_credentials_test.py @@ -13,17 +13,24 @@ # limitations under the License. import os +import sys from typing import Iterator, Union +from unittest import mock import google.auth -import mock import pytest from .user_credentials import main # type: ignore PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"] -MockType = Union[mock.mock.MagicMock, mock.mock.AsyncMock] + +if sys.version_info >= (3, 8): + # Python 3.8+ has an AsyncMock attribute in unittest.mock, but 3.7 does not + MockType = Union[mock.MagicMock, mock.AsyncMock] +else: + # Other definitions and imports + MockType = Union[mock.MagicMock] @pytest.fixture diff --git a/samples/tests/conftest.py b/samples/tests/conftest.py index 2b5b89c43..91603bef2 100644 --- a/samples/tests/conftest.py +++ b/samples/tests/conftest.py @@ -14,10 +14,10 @@ import datetime from typing import Iterator, List +from unittest import mock import uuid import google.auth -import mock import pytest from google.cloud import bigquery diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index 9f71bf11a..28787adb7 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -7,11 +7,16 @@ # Then this file should have foo==1.14.0 db-dtypes==0.3.0 geopandas==0.9.0 -google-api-core==1.31.5 -google-cloud-bigquery-storage==2.6.0 -google-cloud-core==1.6.0 -google-resumable-media==0.6.0 +google-api-core==2.17.1 +google-auth==2.28.1 +google-cloud-bigquery-storage==2.24.0 +google-cloud-core==2.4.1 +google-cloud-testutils==1.4.0 +google-crc32c==1.5.0 +google-resumable-media==2.7.0 +googleapis-common-protos==1.62.0 grpcio==1.47.0 +grpcio-status==1.47.0 ipywidgets==7.7.1 ipython==7.23.1 ipykernel==6.0.0 diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index c2ae78eaa..ebe2d2a7a 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock + import pytest from .helpers import make_client diff --git a/tests/unit/helpers.py b/tests/unit/helpers.py index bc92c0df6..c5414138e 100644 --- a/tests/unit/helpers.py +++ b/tests/unit/helpers.py @@ -12,15 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +from unittest import mock + +import pytest + import google.cloud.bigquery.client import google.cloud.bigquery.dataset -import mock -import pytest def make_connection(*responses): import google.cloud.bigquery._http - import mock from google.cloud.exceptions import NotFound mock_conn = mock.create_autospec(google.cloud.bigquery._http.Connection) diff --git a/tests/unit/job/test_base.py b/tests/unit/job/test_base.py index a61fd3198..186729529 100644 --- a/tests/unit/job/test_base.py +++ b/tests/unit/job/test_base.py @@ -15,11 +15,11 @@ import copy import http import unittest +from unittest import mock from google.api_core import exceptions import google.api_core.retry from google.api_core.future import polling -import mock import pytest from ..helpers import make_connection diff --git a/tests/unit/job/test_copy.py b/tests/unit/job/test_copy.py index a3b5c70e3..e1bb20db2 100644 --- a/tests/unit/job/test_copy.py +++ b/tests/unit/job/test_copy.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock from ..helpers import make_connection diff --git a/tests/unit/job/test_extract.py b/tests/unit/job/test_extract.py index 8bada51af..76ee72f28 100644 --- a/tests/unit/job/test_extract.py +++ b/tests/unit/job/test_extract.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock from ..helpers import make_connection diff --git a/tests/unit/job/test_load.py b/tests/unit/job/test_load.py index c6bbaa2fb..976fec914 100644 --- a/tests/unit/job/test_load.py +++ b/tests/unit/job/test_load.py @@ -13,8 +13,7 @@ # limitations under the License. import copy - -import mock +from unittest import mock from ..helpers import make_connection diff --git a/tests/unit/job/test_query.py b/tests/unit/job/test_query.py index 776234b5b..37ac7ba5e 100644 --- a/tests/unit/job/test_query.py +++ b/tests/unit/job/test_query.py @@ -17,11 +17,11 @@ import http import textwrap import types +from unittest import mock import freezegun from google.api_core import exceptions import google.api_core.retry -import mock import requests from google.cloud.bigquery.client import _LIST_ROWS_FROM_QUERY_RESULTS_FIELDS @@ -382,11 +382,11 @@ def test__done_or_raise_w_timeout(self): job._done_or_raise(timeout=42) fake_get_results.assert_called_once() - call_args = fake_get_results.call_args - self.assertEqual(call_args.kwargs.get("timeout"), 42) + call_args = fake_get_results.call_args[0][1] + self.assertEqual(call_args.timeout, 600.0) - call_args = fake_reload.call_args - self.assertEqual(call_args.kwargs.get("timeout"), 42) + call_args = fake_reload.call_args[1] + self.assertEqual(call_args["timeout"], 42) def test__done_or_raise_w_timeout_and_longer_internal_api_timeout(self): client = _make_client(project=self.PROJECT) @@ -404,11 +404,11 @@ def test__done_or_raise_w_timeout_and_longer_internal_api_timeout(self): expected_timeout = 5.5 fake_get_results.assert_called_once() - call_args = fake_get_results.call_args - self.assertAlmostEqual(call_args.kwargs.get("timeout"), expected_timeout) + call_args = fake_get_results.call_args[0][1] + self.assertAlmostEqual(call_args.timeout, 600.0) call_args = fake_reload.call_args - self.assertAlmostEqual(call_args.kwargs.get("timeout"), expected_timeout) + self.assertAlmostEqual(call_args[1].get("timeout"), expected_timeout) def test__done_or_raise_w_query_results_error_reload_ok(self): client = _make_client(project=self.PROJECT) diff --git a/tests/unit/job/test_query_pandas.py b/tests/unit/job/test_query_pandas.py index 6189830ff..1473ef283 100644 --- a/tests/unit/job/test_query_pandas.py +++ b/tests/unit/job/test_query_pandas.py @@ -15,8 +15,8 @@ import concurrent.futures import copy import json +from unittest import mock -import mock import pytest diff --git a/tests/unit/test__helpers.py b/tests/unit/test__helpers.py index 7e8d815d2..320c57737 100644 --- a/tests/unit/test__helpers.py +++ b/tests/unit/test__helpers.py @@ -16,11 +16,12 @@ import datetime import decimal import json -import unittest import os -import mock import pytest import packaging +import unittest +from unittest import mock + import google.api_core diff --git a/tests/unit/test__http.py b/tests/unit/test__http.py index 09f6d29d7..fd7ecdc42 100644 --- a/tests/unit/test__http.py +++ b/tests/unit/test__http.py @@ -13,8 +13,8 @@ # limitations under the License. import unittest +from unittest import mock -import mock import requests diff --git a/tests/unit/test__pandas_helpers.py b/tests/unit/test__pandas_helpers.py index 7c83d3ec5..abee39065 100644 --- a/tests/unit/test__pandas_helpers.py +++ b/tests/unit/test__pandas_helpers.py @@ -18,6 +18,7 @@ import functools import operator import queue +from unittest import mock import warnings try: @@ -25,8 +26,6 @@ except ImportError: import importlib_metadata as metadata -import mock - try: import pandas import pandas.api.types @@ -1200,7 +1199,7 @@ def test_dataframe_to_parquet_compression_method(module_under_test): call_args = fake_write_table.call_args assert call_args is not None - assert call_args.kwargs.get("compression") == "ZSTD" + assert call_args[1].get("compression") == "ZSTD" @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") @@ -1635,7 +1634,7 @@ def test_dataframe_to_parquet_dict_sequence_schema(module_under_test): schema.SchemaField("field01", "STRING", mode="REQUIRED"), schema.SchemaField("field02", "BOOL", mode="NULLABLE"), ] - schema_arg = fake_to_arrow.call_args.args[1] + schema_arg = fake_to_arrow.call_args[0][1] assert schema_arg == expected_schema_arg diff --git a/tests/unit/test__versions_helpers.py b/tests/unit/test__versions_helpers.py index afe170e7a..8fa099627 100644 --- a/tests/unit/test__versions_helpers.py +++ b/tests/unit/test__versions_helpers.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest +from unittest import mock -import mock +import pytest try: import pyarrow # type: ignore diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 42581edc1..be8bef03c 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -24,9 +24,9 @@ import json import operator import unittest +from unittest import mock import warnings -import mock import requests import packaging import pytest @@ -8733,9 +8733,9 @@ def test_load_table_from_dataframe_w_schema_arrow_custom_compression(self): parquet_compression="LZ4", ) - call_args = fake_to_parquet.call_args + call_args = fake_to_parquet.call_args[1] assert call_args is not None - assert call_args.kwargs.get("parquet_compression") == "LZ4" + assert call_args.get("parquet_compression") == "LZ4" @unittest.skipIf(pandas is None, "Requires `pandas`") @unittest.skipIf(pyarrow is None, "Requires `pyarrow`") @@ -9498,12 +9498,11 @@ def test__do_resumable_upload_custom_project(self): timeout=mock.ANY, ) - # Check the project ID used in the call to initiate resumable upload. initiation_url = next( ( - call.args[1] + call[0][1] for call in transport.request.call_args_list - if call.args[0] == "POST" and "uploadType=resumable" in call.args[1] + if call[0][0] == "POST" and "uploadType=resumable" in call[0][1] ), None, ) # pragma: NO COVER @@ -9525,7 +9524,7 @@ def test__do_resumable_upload_custom_timeout(self): # The timeout should be applied to all underlying calls. for call_args in transport.request.call_args_list: - assert call_args.kwargs.get("timeout") == 3.14 + assert call_args[1].get("timeout") == 3.14 def test__do_multipart_upload(self): transport = self._make_transport([self._make_response(http.client.OK)]) diff --git a/tests/unit/test_create_dataset.py b/tests/unit/test_create_dataset.py index 8374e6e0a..a2491a812 100644 --- a/tests/unit/test_create_dataset.py +++ b/tests/unit/test_create_dataset.py @@ -12,11 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from unittest import mock + from google.cloud.bigquery.dataset import Dataset, DatasetReference from .helpers import make_connection, dataset_polymorphic, make_client import google.cloud.bigquery.dataset from google.cloud.bigquery.retry import DEFAULT_TIMEOUT -import mock import pytest diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index 423349a51..c0164bc73 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -13,8 +13,8 @@ # limitations under the License. import unittest +from unittest import mock -import mock from google.cloud.bigquery.routine.routine import Routine, RoutineReference import pytest from google.cloud.bigquery.dataset import ( diff --git a/tests/unit/test_dbapi_connection.py b/tests/unit/test_dbapi_connection.py index 67777f923..88378ec98 100644 --- a/tests/unit/test_dbapi_connection.py +++ b/tests/unit/test_dbapi_connection.py @@ -14,8 +14,7 @@ import gc import unittest - -import mock +from unittest import mock try: from google.cloud import bigquery_storage diff --git a/tests/unit/test_dbapi_cursor.py b/tests/unit/test_dbapi_cursor.py index 69d33fe17..e9fd2e3dd 100644 --- a/tests/unit/test_dbapi_cursor.py +++ b/tests/unit/test_dbapi_cursor.py @@ -13,9 +13,9 @@ # limitations under the License. import functools -import mock import operator as op import unittest +from unittest import mock import pytest diff --git a/tests/unit/test_encryption_configuration.py b/tests/unit/test_encryption_configuration.py index f432a903b..cdd944a8f 100644 --- a/tests/unit/test_encryption_configuration.py +++ b/tests/unit/test_encryption_configuration.py @@ -13,7 +13,7 @@ # limitations under the License. import unittest -import mock +from unittest import mock class TestEncryptionConfiguration(unittest.TestCase): diff --git a/tests/unit/test_job_retry.py b/tests/unit/test_job_retry.py index 0e984c8fc..d7049c5ca 100644 --- a/tests/unit/test_job_retry.py +++ b/tests/unit/test_job_retry.py @@ -14,8 +14,8 @@ import datetime import re +from unittest import mock -import mock import pytest import google.api_core.exceptions diff --git a/tests/unit/test_list_datasets.py b/tests/unit/test_list_datasets.py index 6f0b55c5e..4ef99fd86 100644 --- a/tests/unit/test_list_datasets.py +++ b/tests/unit/test_list_datasets.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock + import pytest from google.cloud.bigquery.retry import DEFAULT_TIMEOUT diff --git a/tests/unit/test_list_jobs.py b/tests/unit/test_list_jobs.py index 1db6b5668..edb85af0a 100644 --- a/tests/unit/test_list_jobs.py +++ b/tests/unit/test_list_jobs.py @@ -13,8 +13,8 @@ # limitations under the License. import datetime +from unittest import mock -import mock import pytest from google.cloud.bigquery.retry import DEFAULT_TIMEOUT diff --git a/tests/unit/test_list_projects.py b/tests/unit/test_list_projects.py index 190612b44..5260e5246 100644 --- a/tests/unit/test_list_projects.py +++ b/tests/unit/test_list_projects.py @@ -12,7 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import mock +from unittest import mock + import pytest from google.cloud.bigquery.retry import DEFAULT_TIMEOUT diff --git a/tests/unit/test_magics.py b/tests/unit/test_magics.py index 1511cba9c..4b1aaf14d 100644 --- a/tests/unit/test_magics.py +++ b/tests/unit/test_magics.py @@ -15,11 +15,11 @@ import copy import re from concurrent import futures +from unittest import mock import warnings from google.api_core import exceptions import google.auth.credentials -import mock import pytest from tests.unit.helpers import make_connection from test_utils.imports import maybe_fail_import @@ -513,7 +513,7 @@ def test_bigquery_magic_default_connection_user_agent(): with conn_patch as conn, run_query_patch, default_patch: ip.run_cell_magic("bigquery", "", "SELECT 17 as num") - client_info_arg = conn.call_args.kwargs.get("client_info") + client_info_arg = conn.call_args[1].get("client_info") assert client_info_arg is not None assert client_info_arg.user_agent == "ipython-" + IPython.__version__ @@ -663,7 +663,7 @@ def warning_match(warning): assert len(expected_warnings) == 1 assert len(bqstorage_mock.call_args_list) == 1 - kwargs = bqstorage_mock.call_args_list[0].kwargs + kwargs = bqstorage_mock.call_args_list[0][1] assert kwargs.get("credentials") is mock_credentials client_info = kwargs.get("client_info") assert client_info is not None diff --git a/tests/unit/test_opentelemetry_tracing.py b/tests/unit/test_opentelemetry_tracing.py index 4cc58713c..e96e18c6b 100644 --- a/tests/unit/test_opentelemetry_tracing.py +++ b/tests/unit/test_opentelemetry_tracing.py @@ -15,8 +15,7 @@ import datetime import importlib import sys - -import mock +from unittest import mock try: import opentelemetry diff --git a/tests/unit/test_query.py b/tests/unit/test_query.py index f511bf28d..7c36eb75b 100644 --- a/tests/unit/test_query.py +++ b/tests/unit/test_query.py @@ -15,8 +15,7 @@ import datetime import decimal import unittest - -import mock +from unittest import mock class Test_UDFResource(unittest.TestCase): diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py index 2fcb84e21..6e533c849 100644 --- a/tests/unit/test_retry.py +++ b/tests/unit/test_retry.py @@ -13,8 +13,8 @@ # limitations under the License. import unittest +from unittest import mock -import mock import requests.exceptions diff --git a/tests/unit/test_schema.py b/tests/unit/test_schema.py index 26ec0dfef..b17cd0281 100644 --- a/tests/unit/test_schema.py +++ b/tests/unit/test_schema.py @@ -16,8 +16,8 @@ from google.cloud.bigquery.standard_sql import StandardSqlStructType from google.cloud.bigquery.schema import PolicyTagList import unittest +from unittest import mock -import mock import pytest diff --git a/tests/unit/test_table.py b/tests/unit/test_table.py index 00a7f06e6..0d549120f 100644 --- a/tests/unit/test_table.py +++ b/tests/unit/test_table.py @@ -19,9 +19,9 @@ import time import types import unittest +from unittest import mock import warnings -import mock import pytest try: From 438776321f788e62f2d2f7e74f8ae2825740b1ae Mon Sep 17 00:00:00 2001 From: shollyman Date: Thu, 7 Mar 2024 14:30:13 -0800 Subject: [PATCH 06/10] testing: update CI configurations (#1846) * testing: remove unnecessary prerelease targets This PR does two things: * remove unneeded prerelease-deps configs for removed and nonexisting CI targets * fixes the continuous prerelease-deps-3.12 config --- .../{prerelease-deps-3.11.cfg => prerelease-deps-3.12.cfg} | 2 +- .kokoro/continuous/prerelease-deps-3.8.cfg | 7 ------- .kokoro/continuous/prerelease-deps.cfg | 7 ------- .kokoro/presubmit/prerelease-deps-3.8.cfg | 7 ------- .kokoro/presubmit/prerelease-deps.cfg | 7 ------- 5 files changed, 1 insertion(+), 29 deletions(-) rename .kokoro/continuous/{prerelease-deps-3.11.cfg => prerelease-deps-3.12.cfg} (77%) delete mode 100644 .kokoro/continuous/prerelease-deps-3.8.cfg delete mode 100644 .kokoro/continuous/prerelease-deps.cfg delete mode 100644 .kokoro/presubmit/prerelease-deps-3.8.cfg delete mode 100644 .kokoro/presubmit/prerelease-deps.cfg diff --git a/.kokoro/continuous/prerelease-deps-3.11.cfg b/.kokoro/continuous/prerelease-deps-3.12.cfg similarity index 77% rename from .kokoro/continuous/prerelease-deps-3.11.cfg rename to .kokoro/continuous/prerelease-deps-3.12.cfg index 1e19f1239..ece962a17 100644 --- a/.kokoro/continuous/prerelease-deps-3.11.cfg +++ b/.kokoro/continuous/prerelease-deps-3.12.cfg @@ -3,5 +3,5 @@ # Only run this nox session. env_vars: { key: "NOX_SESSION" - value: "prerelease_deps-3.11" + value: "prerelease_deps-3.12" } diff --git a/.kokoro/continuous/prerelease-deps-3.8.cfg b/.kokoro/continuous/prerelease-deps-3.8.cfg deleted file mode 100644 index fabe3e347..000000000 --- a/.kokoro/continuous/prerelease-deps-3.8.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run this nox session. -env_vars: { - key: "NOX_SESSION" - value: "prerelease_deps-3.8" -} diff --git a/.kokoro/continuous/prerelease-deps.cfg b/.kokoro/continuous/prerelease-deps.cfg deleted file mode 100644 index 3595fb43f..000000000 --- a/.kokoro/continuous/prerelease-deps.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run this nox session. -env_vars: { - key: "NOX_SESSION" - value: "prerelease_deps" -} diff --git a/.kokoro/presubmit/prerelease-deps-3.8.cfg b/.kokoro/presubmit/prerelease-deps-3.8.cfg deleted file mode 100644 index fabe3e347..000000000 --- a/.kokoro/presubmit/prerelease-deps-3.8.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run this nox session. -env_vars: { - key: "NOX_SESSION" - value: "prerelease_deps-3.8" -} diff --git a/.kokoro/presubmit/prerelease-deps.cfg b/.kokoro/presubmit/prerelease-deps.cfg deleted file mode 100644 index 3595fb43f..000000000 --- a/.kokoro/presubmit/prerelease-deps.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Only run this nox session. -env_vars: { - key: "NOX_SESSION" - value: "prerelease_deps" -} From 713ce2c2f6ce9931f67cbbcd63ad436ad336ad26 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Fri, 8 Mar 2024 08:34:40 -0500 Subject: [PATCH 07/10] fix: add google-auth as a direct dependency (#1809) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: add google-auth as a direct dependency * update constraints * fix(deps): Require `google-api-core>=1.34.1, >=2.11.0` * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Lingqing Gan Co-authored-by: Chalmer Lowe Co-authored-by: Owl Bot --- .kokoro/continuous/prerelease-deps.cfg | 7 +++++++ .kokoro/presubmit/prerelease-deps.cfg | 7 +++++++ setup.py | 3 ++- 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 .kokoro/continuous/prerelease-deps.cfg create mode 100644 .kokoro/presubmit/prerelease-deps.cfg diff --git a/.kokoro/continuous/prerelease-deps.cfg b/.kokoro/continuous/prerelease-deps.cfg new file mode 100644 index 000000000..3595fb43f --- /dev/null +++ b/.kokoro/continuous/prerelease-deps.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease_deps" +} diff --git a/.kokoro/presubmit/prerelease-deps.cfg b/.kokoro/presubmit/prerelease-deps.cfg new file mode 100644 index 000000000..3595fb43f --- /dev/null +++ b/.kokoro/presubmit/prerelease-deps.cfg @@ -0,0 +1,7 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Only run this nox session. +env_vars: { + key: "NOX_SESSION" + value: "prerelease_deps" +} diff --git a/setup.py b/setup.py index 9fbc91ecb..5a35f4136 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,8 @@ # NOTE: Maintainers, please do not require google-api-core>=2.x.x # Until this issue is closed # https://github.com/googleapis/google-cloud-python/issues/10566 - "google-api-core >= 1.31.5, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0", + "google-api-core[grpc] >= 1.34.1, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", + "google-auth >= 2.14.1, <3.0.0dev", # NOTE: Maintainers, please do not require google-cloud-core>=2.x.x # Until this issue is closed # https://github.com/googleapis/google-cloud-python/issues/10566 From 6dff50f4fbc5aeb644383a4050dd5ffc05015ffe Mon Sep 17 00:00:00 2001 From: shollyman Date: Fri, 8 Mar 2024 15:43:02 -0800 Subject: [PATCH 08/10] fix: supplementary fix to env-based universe resolution (#1847) * fix: promote env-based universe into client option parsing * lint * add client test * import --------- Co-authored-by: Chalmer Lowe --- google/cloud/bigquery/client.py | 25 ++++++++++++------------- tests/unit/test_client.py | 12 ++++++++++++ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index cb4daa897..408e7e49c 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -249,20 +249,19 @@ def __init__( bq_host = _get_bigquery_host() kw_args["api_endpoint"] = bq_host if bq_host != _DEFAULT_HOST else None client_universe = None - if client_options: - if isinstance(client_options, dict): - client_options = google.api_core.client_options.from_dict( - client_options + if client_options is None: + client_options = {} + if isinstance(client_options, dict): + client_options = google.api_core.client_options.from_dict(client_options) + if client_options.api_endpoint: + api_endpoint = client_options.api_endpoint + kw_args["api_endpoint"] = api_endpoint + else: + client_universe = _get_client_universe(client_options) + if client_universe != _DEFAULT_UNIVERSE: + kw_args["api_endpoint"] = _DEFAULT_HOST_TEMPLATE.replace( + "{UNIVERSE_DOMAIN}", client_universe ) - if client_options.api_endpoint: - api_endpoint = client_options.api_endpoint - kw_args["api_endpoint"] = api_endpoint - else: - client_universe = _get_client_universe(client_options) - if client_universe != _DEFAULT_UNIVERSE: - kw_args["api_endpoint"] = _DEFAULT_HOST_TEMPLATE.replace( - "{UNIVERSE_DOMAIN}", client_universe - ) # Ensure credentials and universe are not in conflict. if hasattr(self, "_credentials") and client_universe is not None: _validate_universe(client_universe, self._credentials) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index be8bef03c..d20712a8a 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -23,6 +23,7 @@ import itertools import json import operator +import os import unittest from unittest import mock import warnings @@ -171,6 +172,17 @@ def test_ctor_w_empty_client_options(self): client._connection.API_BASE_URL, client._connection.DEFAULT_API_ENDPOINT ) + @mock.patch.dict(os.environ, {"GOOGLE_CLOUD_UNIVERSE_DOMAIN": "foo.com"}) + def test_ctor_w_only_env_universe(self): + creds = _make_credentials() + http = object() + client = self._make_one( + project=self.PROJECT, + credentials=creds, + _http=http, + ) + self.assertEqual(client._connection.API_BASE_URL, "https://bigquery.foo.com") + def test_ctor_w_client_options_dict(self): creds = _make_credentials() http = object() From 4a1ff52ade535e349521b2877bcc762d280bd31b Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 11 Mar 2024 16:56:52 +0100 Subject: [PATCH 09/10] chore(deps): update all dependencies (#1835) * chore(deps): update all dependencies * Update samples/magics/requirements-test.txt * Update samples/notebooks/requirements.txt * Update samples/magics/requirements.txt update. * Update samples/desktopapp/requirements-test.txt * Update samples/geography/requirements-test.txt * Update samples/notebooks/requirements-test.txt * Update samples/snippets/requirements-test.txt --------- Co-authored-by: Chalmer Lowe --- samples/desktopapp/requirements-test.txt | 3 ++- samples/desktopapp/requirements.txt | 2 +- samples/geography/requirements-test.txt | 3 ++- samples/geography/requirements.txt | 34 ++++++++++++------------ samples/magics/requirements-test.txt | 3 ++- samples/magics/requirements.txt | 4 +-- samples/notebooks/requirements-test.txt | 3 ++- samples/notebooks/requirements.txt | 6 ++--- samples/snippets/requirements-test.txt | 3 ++- samples/snippets/requirements.txt | 2 +- 10 files changed, 34 insertions(+), 29 deletions(-) diff --git a/samples/desktopapp/requirements-test.txt b/samples/desktopapp/requirements-test.txt index fc926cd7c..99d27b06a 100644 --- a/samples/desktopapp/requirements-test.txt +++ b/samples/desktopapp/requirements-test.txt @@ -1,3 +1,4 @@ google-cloud-testutils==1.4.0 -pytest==7.4.3 +pytest==7.4.4; python_version == '3.7' +pytest==8.1.1; python_version >= '3.8' mock==5.1.0 diff --git a/samples/desktopapp/requirements.txt b/samples/desktopapp/requirements.txt index 8d82d4930..78074bbca 100644 --- a/samples/desktopapp/requirements.txt +++ b/samples/desktopapp/requirements.txt @@ -1,2 +1,2 @@ -google-cloud-bigquery==3.14.1 +google-cloud-bigquery==3.18.0 google-auth-oauthlib==1.2.0 diff --git a/samples/geography/requirements-test.txt b/samples/geography/requirements-test.txt index 7749d1f94..a91fa2d55 100644 --- a/samples/geography/requirements-test.txt +++ b/samples/geography/requirements-test.txt @@ -1,2 +1,3 @@ -pytest==7.4.3 +pytest==7.4.4; python_version == '3.7' +pytest==8.1.1; python_version >= '3.8' mock==5.1.0 diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 47e7cc56e..c85bf06d0 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -1,5 +1,5 @@ -attrs==23.1.0 -certifi==2023.11.17 +attrs==23.2.0 +certifi==2024.2.2 cffi===1.15.1; python_version == '3.7' cffi==1.16.0; python_version >= '3.8' charset-normalizer==3.3.2 @@ -8,43 +8,43 @@ click-plugins==1.1.1 cligj==0.7.2 dataclasses==0.8; python_version < '3.7' db-dtypes==1.2.0 -Fiona==1.9.5 +Fiona==1.9.6 geojson==3.1.0 geopandas===0.10.2; python_version == '3.7' geopandas===0.13.2; python_version == '3.8' -geopandas==0.14.1; python_version >= '3.9' -google-api-core==2.15.0 -google-auth==2.25.2 -google-cloud-bigquery==3.14.1 +geopandas==0.14.3; python_version >= '3.9' +google-api-core==2.17.1 +google-auth==2.28.2 +google-cloud-bigquery==3.18.0 google-cloud-bigquery-storage==2.24.0 google-cloud-core==2.4.1 google-crc32c==1.5.0 google-resumable-media==2.7.0 -googleapis-common-protos==1.62.0 -grpcio==1.60.0 +googleapis-common-protos==1.63.0 +grpcio==1.62.1 idna==3.6 munch==4.0.0 mypy-extensions==1.0.0 -packaging==23.2 +packaging==24.0 pandas===1.3.5; python_version == '3.7' pandas===2.0.3; python_version == '3.8' -pandas==2.1.0; python_version >= '3.9' +pandas==2.2.1; python_version >= '3.9' proto-plus==1.23.0 pyarrow==12.0.1; python_version == '3.7' -pyarrow==14.0.1; python_version >= '3.8' +pyarrow==15.0.1; python_version >= '3.8' pyasn1==0.5.1 pyasn1-modules==0.3.0 pycparser==2.21 -pyparsing==3.1.1 -python-dateutil==2.8.2 -pytz==2023.3.post1 +pyparsing==3.1.2 +python-dateutil==2.9.0.post0 +pytz==2024.1 PyYAML==6.0.1 requests==2.31.0 rsa==4.9 -Shapely==2.0.2 +Shapely==2.0.3 six==1.16.0 typing-extensions===4.7.1; python_version == '3.7' -typing-extensions==4.9.0; python_version >= '3.8' +typing-extensions==4.10.0; python_version >= '3.8' typing-inspect==0.9.0 urllib3===1.26.18; python_version == '3.7' urllib3==2.2.1; python_version >= '3.8' diff --git a/samples/magics/requirements-test.txt b/samples/magics/requirements-test.txt index fc926cd7c..99d27b06a 100644 --- a/samples/magics/requirements-test.txt +++ b/samples/magics/requirements-test.txt @@ -1,3 +1,4 @@ google-cloud-testutils==1.4.0 -pytest==7.4.3 +pytest==7.4.4; python_version == '3.7' +pytest==8.1.1; python_version >= '3.8' mock==5.1.0 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index 869d3b4d5..ea987358f 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -1,9 +1,9 @@ db-dtypes==1.2.0 -google.cloud.bigquery==3.14.1 +google.cloud.bigquery==3.18.0 google-cloud-bigquery-storage==2.24.0 ipython===7.31.1; python_version == '3.7' ipython===8.0.1; python_version == '3.8' ipython==8.18.1; python_version >= '3.9' pandas===1.3.5; python_version == '3.7' pandas===2.0.3; python_version == '3.8' -pandas==2.1.0; python_version >= '3.9' +pandas==2.2.1; python_version >= '3.9' diff --git a/samples/notebooks/requirements-test.txt b/samples/notebooks/requirements-test.txt index fc926cd7c..99d27b06a 100644 --- a/samples/notebooks/requirements-test.txt +++ b/samples/notebooks/requirements-test.txt @@ -1,3 +1,4 @@ google-cloud-testutils==1.4.0 -pytest==7.4.3 +pytest==7.4.4; python_version == '3.7' +pytest==8.1.1; python_version >= '3.8' mock==5.1.0 diff --git a/samples/notebooks/requirements.txt b/samples/notebooks/requirements.txt index e8839e1fe..5ce95818e 100644 --- a/samples/notebooks/requirements.txt +++ b/samples/notebooks/requirements.txt @@ -1,12 +1,12 @@ db-dtypes==1.2.0 -google-cloud-bigquery==3.14.1 +google-cloud-bigquery==3.18.0 google-cloud-bigquery-storage==2.24.0 ipython===7.31.1; python_version == '3.7' ipython===8.0.1; python_version == '3.8' ipython==8.18.1; python_version >= '3.9' matplotlib===3.5.3; python_version == '3.7' matplotlib===3.7.4; python_version == '3.8' -matplotlib==3.8.2; python_version >= '3.9' +matplotlib==3.8.3; python_version >= '3.9' pandas===1.3.5; python_version == '3.7' pandas===2.0.3; python_version == '3.8' -pandas==2.1.0; python_version >= '3.9' +pandas==2.2.1; python_version >= '3.9' diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index fc926cd7c..99d27b06a 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,3 +1,4 @@ google-cloud-testutils==1.4.0 -pytest==7.4.3 +pytest==7.4.4; python_version == '3.7' +pytest==8.1.1; python_version >= '3.8' mock==5.1.0 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 365d584c7..fc0a2ef36 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-bigquery==3.14.1 \ No newline at end of file +google-cloud-bigquery==3.18.0 \ No newline at end of file From 641a712766bf68d2fa94467577845b5d07e7b1eb Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 11 Mar 2024 12:30:59 -0400 Subject: [PATCH 10/10] chore(main): release 3.19.0 (#1840) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 17 +++++++++++++++++ google/cloud/bigquery/version.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 350787512..4cb0e1d20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,23 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [3.19.0](https://github.com/googleapis/python-bigquery/compare/v3.18.0...v3.19.0) (2024-03-11) + + +### Features + +* Support RANGE query parameters ([#1827](https://github.com/googleapis/python-bigquery/issues/1827)) ([b359a9a](https://github.com/googleapis/python-bigquery/commit/b359a9a55936a759a36aa69c5e5b014685e1fca6)) +* Support range sql ([#1807](https://github.com/googleapis/python-bigquery/issues/1807)) ([86a45c9](https://github.com/googleapis/python-bigquery/commit/86a45c989836b34dca456bac014352e55d6f86c0)) + + +### Bug Fixes + +* Add google-auth as a direct dependency ([713ce2c](https://github.com/googleapis/python-bigquery/commit/713ce2c2f6ce9931f67cbbcd63ad436ad336ad26)) +* Augment universe_domain handling ([#1837](https://github.com/googleapis/python-bigquery/issues/1837)) ([53c2cbf](https://github.com/googleapis/python-bigquery/commit/53c2cbf98d2961f553747514de273bcd5c117f0e)) +* **deps:** Require google-api-core>=1.34.1, >=2.11.0 ([713ce2c](https://github.com/googleapis/python-bigquery/commit/713ce2c2f6ce9931f67cbbcd63ad436ad336ad26)) +* Supplementary fix to env-based universe resolution ([#1844](https://github.com/googleapis/python-bigquery/issues/1844)) ([b818992](https://github.com/googleapis/python-bigquery/commit/b8189929b6008f7780214822062f8ed05d8d2a01)) +* Supplementary fix to env-based universe resolution ([#1847](https://github.com/googleapis/python-bigquery/issues/1847)) ([6dff50f](https://github.com/googleapis/python-bigquery/commit/6dff50f4fbc5aeb644383a4050dd5ffc05015ffe)) + ## [3.18.0](https://github.com/googleapis/python-bigquery/compare/v3.17.2...v3.18.0) (2024-02-29) diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index 89024cc08..27f24bd19 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.18.0" +__version__ = "3.19.0"