diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index d8a1bbca7..e4e943e02 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:5ea6d0ab82c956b50962f91d94e206d3921537ae5fe1549ec5326381d8905cfa -# created: 2024-01-15T16:32:08.142785673Z + digest: sha256:98f3afd11308259de6e828e37376d18867fd321aba07826e29e4f8d9cab56bad +# created: 2024-02-27T15:56:18.442440378Z diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index bb3d6ca38..bda8e38c4 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -93,30 +93,39 @@ colorlog==6.7.0 \ # via # gcp-docuploader # nox -cryptography==41.0.6 \ - --hash=sha256:068bc551698c234742c40049e46840843f3d98ad7ce265fd2bd4ec0d11306596 \ - --hash=sha256:0f27acb55a4e77b9be8d550d762b0513ef3fc658cd3eb15110ebbcbd626db12c \ - --hash=sha256:2132d5865eea673fe6712c2ed5fb4fa49dba10768bb4cc798345748380ee3660 \ - --hash=sha256:3288acccef021e3c3c10d58933f44e8602cf04dba96d9796d70d537bb2f4bbc4 \ - --hash=sha256:35f3f288e83c3f6f10752467c48919a7a94b7d88cc00b0668372a0d2ad4f8ead \ - --hash=sha256:398ae1fc711b5eb78e977daa3cbf47cec20f2c08c5da129b7a296055fbb22aed \ - --hash=sha256:422e3e31d63743855e43e5a6fcc8b4acab860f560f9321b0ee6269cc7ed70cc3 \ - --hash=sha256:48783b7e2bef51224020efb61b42704207dde583d7e371ef8fc2a5fb6c0aabc7 \ - --hash=sha256:4d03186af98b1c01a4eda396b137f29e4e3fb0173e30f885e27acec8823c1b09 \ - --hash=sha256:5daeb18e7886a358064a68dbcaf441c036cbdb7da52ae744e7b9207b04d3908c \ - --hash=sha256:60e746b11b937911dc70d164060d28d273e31853bb359e2b2033c9e93e6f3c43 \ - --hash=sha256:742ae5e9a2310e9dade7932f9576606836ed174da3c7d26bc3d3ab4bd49b9f65 \ - --hash=sha256:7e00fb556bda398b99b0da289ce7053639d33b572847181d6483ad89835115f6 \ - --hash=sha256:85abd057699b98fce40b41737afb234fef05c67e116f6f3650782c10862c43da \ - --hash=sha256:8efb2af8d4ba9dbc9c9dd8f04d19a7abb5b49eab1f3694e7b5a16a5fc2856f5c \ - --hash=sha256:ae236bb8760c1e55b7a39b6d4d32d2279bc6c7c8500b7d5a13b6fb9fc97be35b \ - --hash=sha256:afda76d84b053923c27ede5edc1ed7d53e3c9f475ebaf63c68e69f1403c405a8 \ - --hash=sha256:b27a7fd4229abef715e064269d98a7e2909ebf92eb6912a9603c7e14c181928c \ - --hash=sha256:b648fe2a45e426aaee684ddca2632f62ec4613ef362f4d681a9a6283d10e079d \ - --hash=sha256:c5a550dc7a3b50b116323e3d376241829fd326ac47bc195e04eb33a8170902a9 \ - --hash=sha256:da46e2b5df770070412c46f87bac0849b8d685c5f2679771de277a422c7d0b86 \ - --hash=sha256:f39812f70fc5c71a15aa3c97b2bbe213c3f2a460b79bd21c40d033bb34a9bf36 \ - --hash=sha256:ff369dd19e8fe0528b02e8df9f2aeb2479f89b1270d90f96a63500afe9af5cae +cryptography==42.0.4 \ + --hash=sha256:01911714117642a3f1792c7f376db572aadadbafcd8d75bb527166009c9f1d1b \ + --hash=sha256:0e89f7b84f421c56e7ff69f11c441ebda73b8a8e6488d322ef71746224c20fce \ + --hash=sha256:12d341bd42cdb7d4937b0cabbdf2a94f949413ac4504904d0cdbdce4a22cbf88 \ + --hash=sha256:15a1fb843c48b4a604663fa30af60818cd28f895572386e5f9b8a665874c26e7 \ + --hash=sha256:1cdcdbd117681c88d717437ada72bdd5be9de117f96e3f4d50dab3f59fd9ab20 \ + --hash=sha256:1df6fcbf60560d2113b5ed90f072dc0b108d64750d4cbd46a21ec882c7aefce9 \ + --hash=sha256:3c6048f217533d89f2f8f4f0fe3044bf0b2090453b7b73d0b77db47b80af8dff \ + --hash=sha256:3e970a2119507d0b104f0a8e281521ad28fc26f2820687b3436b8c9a5fcf20d1 \ + --hash=sha256:44a64043f743485925d3bcac548d05df0f9bb445c5fcca6681889c7c3ab12764 \ + --hash=sha256:4e36685cb634af55e0677d435d425043967ac2f3790ec652b2b88ad03b85c27b \ + --hash=sha256:5f8907fcf57392cd917892ae83708761c6ff3c37a8e835d7246ff0ad251d9298 \ + --hash=sha256:69b22ab6506a3fe483d67d1ed878e1602bdd5912a134e6202c1ec672233241c1 \ + --hash=sha256:6bfadd884e7280df24d26f2186e4e07556a05d37393b0f220a840b083dc6a824 \ + --hash=sha256:6d0fbe73728c44ca3a241eff9aefe6496ab2656d6e7a4ea2459865f2e8613257 \ + --hash=sha256:6ffb03d419edcab93b4b19c22ee80c007fb2d708429cecebf1dd3258956a563a \ + --hash=sha256:810bcf151caefc03e51a3d61e53335cd5c7316c0a105cc695f0959f2c638b129 \ + --hash=sha256:831a4b37accef30cccd34fcb916a5d7b5be3cbbe27268a02832c3e450aea39cb \ + --hash=sha256:887623fe0d70f48ab3f5e4dbf234986b1329a64c066d719432d0698522749929 \ + --hash=sha256:a0298bdc6e98ca21382afe914c642620370ce0470a01e1bef6dd9b5354c36854 \ + --hash=sha256:a1327f280c824ff7885bdeef8578f74690e9079267c1c8bd7dc5cc5aa065ae52 \ + --hash=sha256:c1f25b252d2c87088abc8bbc4f1ecbf7c919e05508a7e8628e6875c40bc70923 \ + --hash=sha256:c3a5cbc620e1e17009f30dd34cb0d85c987afd21c41a74352d1719be33380885 \ + --hash=sha256:ce8613beaffc7c14f091497346ef117c1798c202b01153a8cc7b8e2ebaaf41c0 \ + --hash=sha256:d2a27aca5597c8a71abbe10209184e1a8e91c1fd470b5070a2ea60cafec35bcd \ + --hash=sha256:dad9c385ba8ee025bb0d856714f71d7840020fe176ae0229de618f14dae7a6e2 \ + --hash=sha256:db4b65b02f59035037fde0998974d84244a64c3265bdef32a827ab9b63d61b18 \ + --hash=sha256:e09469a2cec88fb7b078e16d4adec594414397e8879a4341c6ace96013463d5b \ + --hash=sha256:e53dc41cda40b248ebc40b83b31516487f7db95ab8ceac1f042626bc43a2f992 \ + --hash=sha256:f1e85a178384bf19e36779d91ff35c7617c885da487d689b05c1366f9933ad74 \ + --hash=sha256:f47be41843200f7faec0683ad751e5ef11b9a56a220d57f300376cd8aba81660 \ + --hash=sha256:fb0cef872d8193e487fc6bdb08559c3aa41b659a7d9be48b2e10747f47863925 \ + --hash=sha256:ffc73996c4fca3d2b6c1c8c12bfd3ad00def8621da24f547626bf06441400449 # via # gcp-releasetool # secretstorage diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bedb5cf6..350787512 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,25 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [3.18.0](https://github.com/googleapis/python-bigquery/compare/v3.17.2...v3.18.0) (2024-02-29) + + +### Features + +* Support nullable boolean and Int64 dtypes in `insert_rows_from_dataframe` ([#1816](https://github.com/googleapis/python-bigquery/issues/1816)) ([ab0cf4c](https://github.com/googleapis/python-bigquery/commit/ab0cf4cc03292f62b56a8813cfb7681daa87f872)) +* Support slot_ms in QueryPlanEntry ([#1831](https://github.com/googleapis/python-bigquery/issues/1831)) ([d62cabb](https://github.com/googleapis/python-bigquery/commit/d62cabbf115637ecbaf8cc378f39329a5ae74c26)) + + +### Bug Fixes + +* Keyword rendering and docstring improvements ([#1829](https://github.com/googleapis/python-bigquery/issues/1829)) ([4dfb920](https://github.com/googleapis/python-bigquery/commit/4dfb920b106784e98f343b3e3fc8e8ff70c50560)) + + +### Documentation + +* **samples:** Updates to urllib3 constraint for Python 3.7 ([#1834](https://github.com/googleapis/python-bigquery/issues/1834)) ([b099c32](https://github.com/googleapis/python-bigquery/commit/b099c32a83946a347560f6a71d08c3f263e56cb6)) +* Update `client_query_w_named_params.py` to use `query_and_wait` API ([#1782](https://github.com/googleapis/python-bigquery/issues/1782)) ([89dfcb6](https://github.com/googleapis/python-bigquery/commit/89dfcb6469d22e78003a70371a0938a6856e033c)) + ## [3.17.2](https://github.com/googleapis/python-bigquery/compare/v3.17.1...v3.17.2) (2024-01-30) diff --git a/google/cloud/bigquery/_job_helpers.py b/google/cloud/bigquery/_job_helpers.py index 6debc377b..0692c9b65 100644 --- a/google/cloud/bigquery/_job_helpers.py +++ b/google/cloud/bigquery/_job_helpers.py @@ -54,7 +54,7 @@ # The purpose of _TIMEOUT_BUFFER_MILLIS is to allow the server-side timeout to -# happen before the client-side timeout. This is not strictly neccessary, as the +# happen before the client-side timeout. This is not strictly necessary, as the # client retries client-side timeouts, but the hope by making the server-side # timeout slightly shorter is that it can save the server from some unncessary # processing time. diff --git a/google/cloud/bigquery/_pandas_helpers.py b/google/cloud/bigquery/_pandas_helpers.py index bcc869f15..e97dda7e5 100644 --- a/google/cloud/bigquery/_pandas_helpers.py +++ b/google/cloud/bigquery/_pandas_helpers.py @@ -958,6 +958,25 @@ def dataframe_to_json_generator(dataframe): # considered a NaN, however. if isinstance(is_nan, bool) and is_nan: continue + + # Convert numpy types to corresponding Python types. + # https://stackoverflow.com/a/60441783/101923 + if isinstance(value, numpy.bool_): + value = bool(value) + elif isinstance( + value, + ( + numpy.int64, + numpy.int32, + numpy.int16, + numpy.int8, + numpy.uint64, + numpy.uint32, + numpy.uint16, + numpy.uint8, + ), + ): + value = int(value) output[column] = value yield output diff --git a/google/cloud/bigquery/client.py b/google/cloud/bigquery/client.py index 4708e753b..a871dc003 100644 --- a/google/cloud/bigquery/client.py +++ b/google/cloud/bigquery/client.py @@ -1716,20 +1716,24 @@ def delete_job_metadata( :func:`~google.cloud.bigquery.client.Client.cancel_job` instead. Args: - job_id: Job or job identifier. - - Keyword Arguments: - project: + job_id (Union[ \ + str, \ + LoadJob, \ + CopyJob, \ + ExtractJob, \ + QueryJob \ + ]): Job or job identifier. + project (Optional[str]): ID of the project which owns the job (defaults to the client's project). - location: + location (Optional[str]): Location where the job was run. Ignored if ``job_id`` is a job object. - retry: + retry (Optional[google.api_core.retry.Retry]): How to retry the RPC. - timeout: + timeout (Optional[float]): The number of seconds to wait for the underlying HTTP transport before using ``retry``. - not_found_ok: + not_found_ok (Optional[bool]): Defaults to ``False``. If ``True``, ignore "not found" errors when deleting the job. """ @@ -1970,12 +1974,10 @@ def create_job( timeout: TimeoutType = DEFAULT_TIMEOUT, ) -> Union[job.LoadJob, job.CopyJob, job.ExtractJob, job.QueryJob]: """Create a new job. + Args: job_config (dict): configuration job representation returned from the API. - - Keyword Arguments: - retry (Optional[google.api_core.retry.Retry]): - How to retry the RPC. + retry (Optional[google.api_core.retry.Retry]): How to retry the RPC. timeout (Optional[float]): The number of seconds to wait for the underlying HTTP transport before using ``retry``. @@ -2066,10 +2068,14 @@ def get_job( https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs/get Args: - job_id: + job_id (Union[ \ + str, \ + job.LoadJob, \ + job.CopyJob, \ + job.ExtractJob, \ + job.QueryJob \ + ]): Job identifier. - - Keyword Arguments: project (Optional[str]): ID of the project which owns the job (defaults to the client's project). location (Optional[str]): @@ -2137,8 +2143,6 @@ def cancel_job( google.cloud.bigquery.job.ExtractJob, \ google.cloud.bigquery.job.QueryJob \ ]): Job identifier. - - Keyword Arguments: project (Optional[str]): ID of the project which owns the job (defaults to the client's project). location (Optional[str]): @@ -2340,8 +2344,6 @@ def load_table_from_uri( in, this method attempts to create a table reference from a string using :func:`google.cloud.bigquery.table.TableReference.from_string`. - - Keyword Arguments: job_id (Optional[str]): Name of the job. job_id_prefix (Optional[str]): The user-provided prefix for a randomly generated job ID. @@ -2415,39 +2417,42 @@ def load_table_from_file( returns a :class:`~google.cloud.bigquery.job.LoadJob`. Args: - file_obj: + file_obj (IO[bytes]): A file handle opened in binary mode for reading. - destination: + destination (Union[Table, \ + TableReference, \ + TableListItem, \ + str \ + ]): Table into which data is to be loaded. If a string is passed in, this method attempts to create a table reference from a string using :func:`google.cloud.bigquery.table.TableReference.from_string`. - - Keyword Arguments: - rewind: + rewind (Optional[bool]): If True, seek to the beginning of the file handle before - reading the file. - size: + reading the file. Defaults to False. + size (Optional[int]): The number of bytes to read from the file handle. If size is ``None`` or large, resumable upload will be used. Otherwise, multipart upload will be used. - num_retries: Number of upload retries. Defaults to 6. - job_id: Name of the job. - job_id_prefix: + num_retries (Optional[int]): Number of upload retries. Defaults to 6. + job_id (Optional[str]): Name of the job. + job_id_prefix (Optional[str]): The user-provided prefix for a randomly generated job ID. This parameter will be ignored if a ``job_id`` is also given. - location: + location (Optional[str]): Location where to run the job. Must match the location of the destination table. - project: + project (Optional[str]): Project ID of the project of where to run the job. Defaults to the client's project. - job_config: + job_config (Optional[LoadJobConfig]): Extra configuration options for the job. - timeout: + timeout (Optional[float]): The number of seconds to wait for the underlying HTTP transport before using ``retry``. Depending on the retry strategy, a request may be repeated several times using the same timeout each time. + Defaults to None. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. @@ -2535,9 +2540,13 @@ def load_table_from_dataframe( https://github.com/googleapis/python-bigquery/issues/19 Args: - dataframe: + dataframe (pandas.Dataframe): A :class:`~pandas.DataFrame` containing the data to load. - destination: + destination (Union[ \ + Table, \ + TableReference, \ + str \ + ]): The destination table to use for loading the data. If it is an existing table, the schema of the :class:`~pandas.DataFrame` must match the schema of the destination table. If the table @@ -2547,21 +2556,19 @@ def load_table_from_dataframe( If a string is passed in, this method attempts to create a table reference from a string using :func:`google.cloud.bigquery.table.TableReference.from_string`. - - Keyword Arguments: - num_retries: Number of upload retries. - job_id: Name of the job. - job_id_prefix: + num_retries (Optional[int]): Number of upload retries. Defaults to 6. + job_id (Optional[str]): Name of the job. + job_id_prefix (Optional[str]): The user-provided prefix for a randomly generated job ID. This parameter will be ignored if a ``job_id`` is also given. - location: + location (Optional[str]): Location where to run the job. Must match the location of the destination table. - project: + project (Optional[str]): Project ID of the project of where to run the job. Defaults to the client's project. - job_config: + job_config (Optional[LoadJobConfig]): Extra configuration options for the job. To override the default pandas data type conversions, supply @@ -2578,9 +2585,10 @@ def load_table_from_dataframe( :attr:`~google.cloud.bigquery.job.SourceFormat.CSV` and :attr:`~google.cloud.bigquery.job.SourceFormat.PARQUET` are supported. - parquet_compression: + parquet_compression (Optional[str]): [Beta] The compression method to use if intermittently serializing ``dataframe`` to a parquet file. + Defaults to "snappy". The argument is directly passed as the ``compression`` argument to the underlying ``pyarrow.parquet.write_table()`` @@ -2591,10 +2599,11 @@ def load_table_from_dataframe( passed as the ``compression`` argument to the underlying ``DataFrame.to_parquet()`` method. https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_parquet.html#pandas.DataFrame.to_parquet - timeout: + timeout (Optional[flaot]): The number of seconds to wait for the underlying HTTP transport before using ``retry``. Depending on the retry strategy, a request may be repeated several times using the same timeout each time. + Defaults to None. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. @@ -2784,32 +2793,36 @@ def load_table_from_json( client = bigquery.Client() client.load_table_from_file(data_as_file, ...) - destination: + destination (Union[ \ + Table, \ + TableReference, \ + TableListItem, \ + str \ + ]): Table into which data is to be loaded. If a string is passed in, this method attempts to create a table reference from a string using :func:`google.cloud.bigquery.table.TableReference.from_string`. - - Keyword Arguments: - num_retries: Number of upload retries. - job_id: Name of the job. - job_id_prefix: + num_retries (Optional[int]): Number of upload retries. Defaults to 6. + job_id (Optional[str]): Name of the job. + job_id_prefix (Optional[str]): The user-provided prefix for a randomly generated job ID. This parameter will be ignored if a ``job_id`` is also given. - location: + location (Optional[str]): Location where to run the job. Must match the location of the destination table. - project: + project (Optional[str]): Project ID of the project of where to run the job. Defaults to the client's project. - job_config: + job_config (Optional[LoadJobConfig]): Extra configuration options for the job. The ``source_format`` setting is always set to :attr:`~google.cloud.bigquery.job.SourceFormat.NEWLINE_DELIMITED_JSON`. - timeout: + timeout (Optional[float]): The number of seconds to wait for the underlying HTTP transport before using ``retry``. Depending on the retry strategy, a request may be repeated several times using the same timeout each time. + Defaults to None. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. @@ -2885,23 +2898,19 @@ def _do_resumable_upload( """Perform a resumable upload. Args: - stream: A bytes IO object open for reading. - - metadata: The metadata associated with the upload. - - num_retries: + stream (IO[bytes]): A bytes IO object open for reading. + metadata (Mapping[str, str]): The metadata associated with the upload. + num_retries (int): Number of upload retries. (Deprecated: This argument will be removed in a future release.) - - timeout: + timeout (Optional[float]): The number of seconds to wait for the underlying HTTP transport before using ``retry``. Depending on the retry strategy, a request may be repeated several times using the same timeout each time. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. - - project: + project (Optional[str]): Project ID of the project of where to run the upload. Defaults to the client's project. @@ -2929,23 +2938,19 @@ def _initiate_resumable_upload( """Initiate a resumable upload. Args: - stream: A bytes IO object open for reading. - - metadata: The metadata associated with the upload. - - num_retries: + stream (IO[bytes]): A bytes IO object open for reading. + metadata (Mapping[str, str]): The metadata associated with the upload. + num_retries (int): Number of upload retries. (Deprecated: This argument will be removed in a future release.) - - timeout: + timeout (Optional[float]): The number of seconds to wait for the underlying HTTP transport before using ``retry``. Depending on the retry strategy, a request may be repeated several times using the same timeout each time. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. - - project: + project (Optional[str]): Project ID of the project of where to run the upload. Defaults to the client's project. @@ -3005,28 +3010,23 @@ def _do_multipart_upload( """Perform a multipart upload. Args: - stream: A bytes IO object open for reading. - - metadata: The metadata associated with the upload. - - size: + stream (IO[bytes]): A bytes IO object open for reading. + metadata (Mapping[str, str]): The metadata associated with the upload. + size (int): The number of bytes to be uploaded (which will be read from ``stream``). If not provided, the upload will be concluded once ``stream`` is exhausted (or :data:`None`). - - num_retries: + num_retries (int): Number of upload retries. (Deprecated: This argument will be removed in a future release.) - - timeout: + timeout (Optional[float]): The number of seconds to wait for the underlying HTTP transport before using ``retry``. Depending on the retry strategy, a request may be repeated several times using the same timeout each time. Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. - - project: + project (Optional[str]): Project ID of the project of where to run the upload. Defaults to the client's project. @@ -3118,8 +3118,6 @@ def copy_table( str, \ ]): Table into which data is to be copied. - - Keyword Arguments: job_id (Optional[str]): The ID of the job. job_id_prefix (Optional[str]): The user-provided prefix for a randomly generated job ID. @@ -3216,8 +3214,6 @@ def extract_table( URIs of Cloud Storage file(s) into which table data is to be extracted; in format ``gs:///``. - - Keyword Arguments: job_id (Optional[str]): The ID of the job. job_id_prefix (Optional[str]): The user-provided prefix for a randomly generated job ID. @@ -3306,8 +3302,6 @@ def query( query (str): SQL query to be executed. Defaults to the standard SQL dialect. Use the ``job_config`` parameter to change dialects. - - Keyword Arguments: job_config (Optional[google.cloud.bigquery.job.QueryJobConfig]): Extra configuration options for the job. To override any options that were previously set in diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index ac0c51973..e45a46894 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -2263,6 +2263,11 @@ def steps(self): for step in self._properties.get("steps", []) ] + @property + def slot_ms(self): + """Optional[int]: Slot-milliseconds used by the stage.""" + return _helpers._int_or_none(self._properties.get("slotMs")) + class TimelineEntry(object): """TimelineEntry represents progress of a query job at a particular diff --git a/google/cloud/bigquery/magics/magics.py b/google/cloud/bigquery/magics/magics.py index b7c685d9a..8464c8792 100644 --- a/google/cloud/bigquery/magics/magics.py +++ b/google/cloud/bigquery/magics/magics.py @@ -288,7 +288,7 @@ def _handle_error(error, destination_var=None): Args: error (Exception): - An exception that ocurred during the query execution. + An exception that occurred during the query execution. destination_var (Optional[str]): The name of the IPython session variable to store the query job. """ diff --git a/google/cloud/bigquery/opentelemetry_tracing.py b/google/cloud/bigquery/opentelemetry_tracing.py index be02c1686..e2a05e4d0 100644 --- a/google/cloud/bigquery/opentelemetry_tracing.py +++ b/google/cloud/bigquery/opentelemetry_tracing.py @@ -90,7 +90,7 @@ def _get_final_span_attributes(attributes=None, client=None, job_ref=None): """Compiles attributes from: client, job_ref, user-provided attributes. Attributes from all of these sources are merged together. Note the - attributes are added sequentially based on perceived order of precendence: + attributes are added sequentially based on perceived order of precedence: i.e. attributes added last may overwrite attributes added earlier. Args: diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index 771b77a38..89024cc08 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.17.2" +__version__ = "3.18.0" diff --git a/samples/client_query_w_named_params.py b/samples/client_query_w_named_params.py index 6dd72d44f..b7e59a81a 100644 --- a/samples/client_query_w_named_params.py +++ b/samples/client_query_w_named_params.py @@ -33,8 +33,10 @@ def client_query_w_named_params() -> None: bigquery.ScalarQueryParameter("min_word_count", "INT64", 250), ] ) - query_job = client.query(query, job_config=job_config) # Make an API request. + results = client.query_and_wait( + query, job_config=job_config + ) # Make an API request. - for row in query_job: + for row in results: print("{}: \t{}".format(row.word, row.word_count)) # [END bigquery_query_params_named] diff --git a/samples/desktopapp/requirements-test.txt b/samples/desktopapp/requirements-test.txt index 514f09705..fc926cd7c 100644 --- a/samples/desktopapp/requirements-test.txt +++ b/samples/desktopapp/requirements-test.txt @@ -1,3 +1,3 @@ -google-cloud-testutils==1.3.3 -pytest==7.4.0 +google-cloud-testutils==1.4.0 +pytest==7.4.3 mock==5.1.0 diff --git a/samples/desktopapp/requirements.txt b/samples/desktopapp/requirements.txt index a5b3ad130..8d82d4930 100644 --- a/samples/desktopapp/requirements.txt +++ b/samples/desktopapp/requirements.txt @@ -1,2 +1,2 @@ -google-cloud-bigquery==3.11.4 -google-auth-oauthlib==1.0.0 +google-cloud-bigquery==3.14.1 +google-auth-oauthlib==1.2.0 diff --git a/samples/geography/requirements-test.txt b/samples/geography/requirements-test.txt index 6585a560a..7749d1f94 100644 --- a/samples/geography/requirements-test.txt +++ b/samples/geography/requirements-test.txt @@ -1,2 +1,2 @@ -pytest==7.4.0 +pytest==7.4.3 mock==5.1.0 diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index d6cea7ec5..47e7cc56e 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -1,48 +1,50 @@ attrs==23.1.0 -certifi==2023.7.22 -cffi==1.15.1 -charset-normalizer==3.2.0 -click==8.1.6 +certifi==2023.11.17 +cffi===1.15.1; python_version == '3.7' +cffi==1.16.0; python_version >= '3.8' +charset-normalizer==3.3.2 +click==8.1.7 click-plugins==1.1.1 cligj==0.7.2 dataclasses==0.8; python_version < '3.7' -db-dtypes==1.1.1 +db-dtypes==1.2.0 Fiona==1.9.5 geojson==3.1.0 geopandas===0.10.2; python_version == '3.7' -geopandas==0.13.2; python_version == '3.8' +geopandas===0.13.2; python_version == '3.8' geopandas==0.14.1; python_version >= '3.9' -google-api-core==2.11.1 -google-auth==2.22.0 -google-cloud-bigquery==3.11.4 -google-cloud-bigquery-storage==2.22.0 -google-cloud-core==2.3.3 +google-api-core==2.15.0 +google-auth==2.25.2 +google-cloud-bigquery==3.14.1 +google-cloud-bigquery-storage==2.24.0 +google-cloud-core==2.4.1 google-crc32c==1.5.0 -google-resumable-media==2.5.0 -googleapis-common-protos==1.60.0 -grpcio==1.59.0 -idna==3.4 -libcst==1.0.1; python_version == '3.7' -libcst==1.1.0; python_version >= '3.8' +google-resumable-media==2.7.0 +googleapis-common-protos==1.62.0 +grpcio==1.60.0 +idna==3.6 munch==4.0.0 mypy-extensions==1.0.0 -packaging==23.1 +packaging==23.2 pandas===1.3.5; python_version == '3.7' -pandas==2.0.3; python_version >= '3.8' -proto-plus==1.22.3 +pandas===2.0.3; python_version == '3.8' +pandas==2.1.0; python_version >= '3.9' +proto-plus==1.23.0 pyarrow==12.0.1; python_version == '3.7' pyarrow==14.0.1; python_version >= '3.8' -pyasn1==0.5.0 +pyasn1==0.5.1 pyasn1-modules==0.3.0 pycparser==2.21 pyparsing==3.1.1 python-dateutil==2.8.2 -pytz==2023.3 +pytz==2023.3.post1 PyYAML==6.0.1 requests==2.31.0 rsa==4.9 Shapely==2.0.2 six==1.16.0 -typing-extensions==4.7.1 +typing-extensions===4.7.1; python_version == '3.7' +typing-extensions==4.9.0; python_version >= '3.8' typing-inspect==0.9.0 -urllib3==1.26.18 +urllib3===1.26.18; python_version == '3.7' +urllib3==2.2.1; python_version >= '3.8' diff --git a/samples/magics/requirements-test.txt b/samples/magics/requirements-test.txt index 514f09705..fc926cd7c 100644 --- a/samples/magics/requirements-test.txt +++ b/samples/magics/requirements-test.txt @@ -1,3 +1,3 @@ -google-cloud-testutils==1.3.3 -pytest==7.4.0 +google-cloud-testutils==1.4.0 +pytest==7.4.3 mock==5.1.0 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index c8f6b2765..869d3b4d5 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -1,8 +1,9 @@ -db-dtypes==1.1.1 -google.cloud.bigquery==3.11.4 -google-cloud-bigquery-storage==2.22.0 +db-dtypes==1.2.0 +google.cloud.bigquery==3.14.1 +google-cloud-bigquery-storage==2.24.0 ipython===7.31.1; python_version == '3.7' ipython===8.0.1; python_version == '3.8' -ipython==8.14.0; python_version >= '3.9' +ipython==8.18.1; python_version >= '3.9' pandas===1.3.5; python_version == '3.7' -pandas==2.0.3; python_version >= '3.8' +pandas===2.0.3; python_version == '3.8' +pandas==2.1.0; python_version >= '3.9' diff --git a/samples/notebooks/requirements-test.txt b/samples/notebooks/requirements-test.txt index 514f09705..fc926cd7c 100644 --- a/samples/notebooks/requirements-test.txt +++ b/samples/notebooks/requirements-test.txt @@ -1,3 +1,3 @@ -google-cloud-testutils==1.3.3 -pytest==7.4.0 +google-cloud-testutils==1.4.0 +pytest==7.4.3 mock==5.1.0 diff --git a/samples/notebooks/requirements.txt b/samples/notebooks/requirements.txt index 22c46297f..e8839e1fe 100644 --- a/samples/notebooks/requirements.txt +++ b/samples/notebooks/requirements.txt @@ -1,10 +1,12 @@ -db-dtypes==1.1.1 -google-cloud-bigquery==3.11.4 -google-cloud-bigquery-storage==2.22.0 +db-dtypes==1.2.0 +google-cloud-bigquery==3.14.1 +google-cloud-bigquery-storage==2.24.0 ipython===7.31.1; python_version == '3.7' ipython===8.0.1; python_version == '3.8' -ipython==8.14.0; python_version >= '3.9' +ipython==8.18.1; python_version >= '3.9' matplotlib===3.5.3; python_version == '3.7' -matplotlib==3.7.2; python_version >= '3.8' +matplotlib===3.7.4; python_version == '3.8' +matplotlib==3.8.2; python_version >= '3.9' pandas===1.3.5; python_version == '3.7' -pandas==2.0.3; python_version >= '3.8' +pandas===2.0.3; python_version == '3.8' +pandas==2.1.0; python_version >= '3.9' diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index 514f09705..fc926cd7c 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,3 +1,3 @@ -google-cloud-testutils==1.3.3 -pytest==7.4.0 +google-cloud-testutils==1.4.0 +pytest==7.4.3 mock==5.1.0 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index f49c7494f..365d584c7 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1 +1 @@ -google-cloud-bigquery==3.11.4 \ No newline at end of file +google-cloud-bigquery==3.14.1 \ No newline at end of file diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index 2ea482e8b..9f71bf11a 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -27,4 +27,4 @@ python-dateutil==2.7.3 requests==2.21.0 Shapely==1.8.4 six==1.13.0 -tqdm==4.7.4 +tqdm==4.7.4 \ No newline at end of file diff --git a/tests/system/test_pandas.py b/tests/system/test_pandas.py index e93f245c0..85c7b79e6 100644 --- a/tests/system/test_pandas.py +++ b/tests/system/test_pandas.py @@ -835,7 +835,9 @@ def test_insert_rows_from_dataframe(bigquery_client, dataset_id): schema = [ SF("float_col", "FLOAT", mode="REQUIRED"), SF("int_col", "INTEGER", mode="REQUIRED"), + SF("int64_col", "INTEGER", mode="NULLABLE"), SF("bool_col", "BOOLEAN", mode="REQUIRED"), + SF("boolean_col", "BOOLEAN", mode="NULLABLE"), SF("string_col", "STRING", mode="NULLABLE"), SF("date_col", "DATE", mode="NULLABLE"), SF("time_col", "TIME", mode="NULLABLE"), @@ -898,6 +900,15 @@ def test_insert_rows_from_dataframe(bigquery_client, dataset_id): dataframe["date_col"] = dataframe["date_col"].astype("dbdate") dataframe["time_col"] = dataframe["time_col"].astype("dbtime") + # Support nullable integer and boolean dtypes. + # https://github.com/googleapis/python-bigquery/issues/1815 + dataframe["int64_col"] = pandas.Series( + [-11, -22, pandas.NA, -44, -55, -66], dtype="Int64" + ) + dataframe["boolean_col"] = pandas.Series( + [True, False, True, pandas.NA, True, False], dtype="boolean" + ) + table_id = f"{bigquery_client.project}.{dataset_id}.test_insert_rows_from_dataframe" table_arg = bigquery.Table(table_id, schema=schema) table = helpers.retry_403(bigquery_client.create_table)(table_arg) @@ -910,7 +921,7 @@ def test_insert_rows_from_dataframe(bigquery_client, dataset_id): expected = [ # Pandas often represents NULL values as NaN. Convert to None for # easier comparison. - tuple(None if col != col else col for col in data_row) + tuple(None if pandas.isna(col) else col for col in data_row) for data_row in dataframe.itertuples(index=False) ] diff --git a/tests/system/test_query.py b/tests/system/test_query.py index b8e0c00da..82be40693 100644 --- a/tests/system/test_query.py +++ b/tests/system/test_query.py @@ -477,7 +477,7 @@ def test_query_error_w_api_method_default(bigquery_client: bigquery.Client): """Test that an exception is not thrown until fetching the results. For backwards compatibility, jobs.insert is the default API method. With - jobs.insert, a failed query job is "sucessfully" created. An exception is + jobs.insert, a failed query job is "successfully" created. An exception is thrown when fetching the results. """ diff --git a/tests/unit/job/test_query_stats.py b/tests/unit/job/test_query_stats.py index bdd0fb627..61b278d43 100644 --- a/tests/unit/job/test_query_stats.py +++ b/tests/unit/job/test_query_stats.py @@ -261,6 +261,7 @@ class TestQueryPlanEntry(_Base): STATUS = "STATUS" SHUFFLE_OUTPUT_BYTES = 1024 SHUFFLE_OUTPUT_BYTES_SPILLED = 1 + SLOT_MS = 25 START_RFC3339_MICROS = "2018-04-01T00:00:00.000000Z" END_RFC3339_MICROS = "2018-04-01T00:00:04.000000Z" @@ -305,6 +306,7 @@ def test_from_api_repr_empty(self): self.assertIsNone(entry.shuffle_output_bytes) self.assertIsNone(entry.shuffle_output_bytes_spilled) self.assertEqual(entry.steps, []) + self.assertIsNone(entry.slot_ms) def test_from_api_repr_normal(self): from google.cloud.bigquery.job import QueryPlanEntryStep @@ -348,6 +350,7 @@ def test_from_api_repr_normal(self): "substeps": TestQueryPlanEntryStep.SUBSTEPS, } ], + "slotMs": self.SLOT_MS, } klass = self._get_target_class() @@ -366,6 +369,7 @@ def test_from_api_repr_normal(self): self.assertEqual(entry.records_written, self.RECORDS_WRITTEN) self.assertEqual(entry.status, self.STATUS) self.assertEqual(entry.steps, steps) + self.assertEqual(entry.slot_ms, self.SLOT_MS) def test_start(self): from google.cloud._helpers import _RFC3339_MICROS diff --git a/tests/unit/test__job_helpers.py b/tests/unit/test__job_helpers.py index 404a546ff..c30964c57 100644 --- a/tests/unit/test__job_helpers.py +++ b/tests/unit/test__job_helpers.py @@ -711,7 +711,7 @@ def test_query_and_wait_caches_completed_query_results_one_page(): {"f": [{"v": "Phred Phlyntstone"}, {"v": "32"}]}, {"f": [{"v": "Bharney Rhubble"}, {"v": "33"}]}, ], - # Even though totalRows > len(rows), we should use the presense of a + # Even though totalRows > len(rows), we should use the presence of a # next page token to decide if there are any more pages. "totalRows": 8, } @@ -828,7 +828,7 @@ def test_query_and_wait_caches_completed_query_results_more_pages(): {"f": [{"v": "Phred Phlyntstone"}, {"v": "32"}]}, {"f": [{"v": "Bharney Rhubble"}, {"v": "33"}]}, ], - # Even though totalRows <= len(rows), we should use the presense of a + # Even though totalRows <= len(rows), we should use the presence of a # next page token to decide if there are any more pages. "totalRows": 2, "pageToken": "page-2", @@ -981,7 +981,7 @@ def test_query_and_wait_incomplete_query(): {"f": [{"v": "Phred Phlyntstone"}, {"v": "32"}]}, {"f": [{"v": "Bharney Rhubble"}, {"v": "33"}]}, ], - # Even though totalRows <= len(rows), we should use the presense of a + # Even though totalRows <= len(rows), we should use the presence of a # next page token to decide if there are any more pages. "totalRows": 2, "pageToken": "page-2", diff --git a/tests/unit/test__pandas_helpers.py b/tests/unit/test__pandas_helpers.py index ad40a6da6..7c83d3ec5 100644 --- a/tests/unit/test__pandas_helpers.py +++ b/tests/unit/test__pandas_helpers.py @@ -808,29 +808,60 @@ def test_list_columns_and_indexes_with_named_index_same_as_column_name( @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_dataframe_to_json_generator(module_under_test): utcnow = datetime.datetime.utcnow() - df_data = collections.OrderedDict( - [ - ("a_series", [pandas.NA, 2, 3, 4]), - ("b_series", [0.1, float("NaN"), 0.3, 0.4]), - ("c_series", ["a", "b", pandas.NA, "d"]), - ("d_series", [utcnow, utcnow, utcnow, pandas.NaT]), - ("e_series", [True, False, True, None]), - ] - ) dataframe = pandas.DataFrame( - df_data, index=pandas.Index([4, 5, 6, 7], name="a_index") + { + "a_series": [1, 2, 3, 4], + "b_series": [0.1, float("NaN"), 0.3, 0.4], + "c_series": ["a", "b", pandas.NA, "d"], + "d_series": [utcnow, utcnow, utcnow, pandas.NaT], + "e_series": [True, False, True, None], + # Support nullable dtypes. + # https://github.com/googleapis/python-bigquery/issues/1815 + "boolean_series": pandas.Series( + [True, False, pandas.NA, False], dtype="boolean" + ), + "int64_series": pandas.Series([-1, pandas.NA, -3, -4], dtype="Int64"), + } ) - dataframe = dataframe.astype({"a_series": pandas.Int64Dtype()}) + # Index is not included, even if it is not the default and has a name. + dataframe = dataframe.rename(index=lambda idx: idx + 4) + dataframe.index.name = "a_index" - rows = module_under_test.dataframe_to_json_generator(dataframe) + rows = list(module_under_test.dataframe_to_json_generator(dataframe)) expected = [ - {"b_series": 0.1, "c_series": "a", "d_series": utcnow, "e_series": True}, - {"a_series": 2, "c_series": "b", "d_series": utcnow, "e_series": False}, - {"a_series": 3, "b_series": 0.3, "d_series": utcnow, "e_series": True}, - {"a_series": 4, "b_series": 0.4, "c_series": "d"}, + { + "a_series": 1, + "b_series": 0.1, + "c_series": "a", + "d_series": utcnow, + "e_series": True, + "boolean_series": True, + "int64_series": -1, + }, + { + "a_series": 2, + "c_series": "b", + "d_series": utcnow, + "e_series": False, + "boolean_series": False, + }, + { + "a_series": 3, + "b_series": 0.3, + "d_series": utcnow, + "e_series": True, + "int64_series": -3, + }, + { + "a_series": 4, + "b_series": 0.4, + "c_series": "d", + "boolean_series": False, + "int64_series": -4, + }, ] - assert list(rows) == expected + assert rows == expected @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") diff --git a/tests/unit/test_table.py b/tests/unit/test_table.py index e4d0c66ab..00a7f06e6 100644 --- a/tests/unit/test_table.py +++ b/tests/unit/test_table.py @@ -3285,6 +3285,9 @@ def test_to_dataframe_iterable_w_bqstorage(self): # Don't close the client if it was passed in. bqstorage_client._transport.grpc_channel.close.assert_not_called() + @unittest.skipIf( + bigquery_storage is None, "Requires `google-cloud-bigquery-storage`" + ) @unittest.skipIf(pandas is None, "Requires `pandas`") def test_to_dataframe_iterable_w_bqstorage_max_results_warning(self): from google.cloud.bigquery import schema