From 6d84bb22e527f5ee19e9fea44933cc766facab79 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 7 Apr 2025 13:13:15 -0700 Subject: [PATCH 1/6] removed patch script --- docs/scripts/patch_devsite_toc.py | 216 ------------------------------ noxfile.py | 3 - 2 files changed, 219 deletions(-) delete mode 100644 docs/scripts/patch_devsite_toc.py diff --git a/docs/scripts/patch_devsite_toc.py b/docs/scripts/patch_devsite_toc.py deleted file mode 100644 index 5889300d2..000000000 --- a/docs/scripts/patch_devsite_toc.py +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -This script will run after ``nox -s docfx`` is run. docfx is the api doc format used by -google cloud. It is described here: https://github.com/googleapis/docuploader?tab=readme-ov-file#requirements-for-docfx-yaml-tarballs. - -One of the file used by docfx is toc.yml which is used to generate the table of contents sidebar. -This script will patch file to create subfolders for each of the clients -""" - - -import yaml -import os -import shutil - -# set working directory to /docs -os.chdir(f"{os.path.dirname(os.path.abspath(__file__))}/{os.pardir}") - - -def add_sections(toc_file_path, section_list, output_file_path=None): - """ - Add new sections to the autogenerated docfx table of contents file - - Takes in a list of TocSection objects, which should point to a directory of rst files - within the main /docs directory, which represents a self-contained section of content - - :param toc_file_path: path to the autogenerated toc file - :param section_list: list of TocSection objects to add - :param output_file_path: path to save the updated toc file. If None, save to the input file - """ - # remove any sections that are already in the toc - remove_sections(toc_file_path, [section.title for section in section_list]) - # add new sections - current_toc = yaml.safe_load(open(toc_file_path, "r")) - for section in section_list: - print(f"Adding section {section.title}...") - current_toc[0]["items"].insert(-1, section.to_dict()) - section.copy_markdown() - # save file - if output_file_path is None: - output_file_path = toc_file_path - with open(output_file_path, "w") as f: - yaml.dump(current_toc, f) - - -def remove_sections(toc_file_path, section_list, output_file_path=None): - """ - Remove sections from the autogenerated docfx table of contents file - - Takes in a list of string section names to remove from the toc file - - :param toc_file_path: path to the autogenerated toc file - :param section_list: list of section names to remove - :param output_file_path: path to save the updated toc file. If None, save to the input file - """ - current_toc = yaml.safe_load(open(toc_file_path, "r")) - print(f"Removing sections {section_list}...") - new_items = [d for d in current_toc[0]["items"] if d["name"] not in section_list] - current_toc[0]["items"] = new_items - # save file - if output_file_path is None: - output_file_path = toc_file_path - with open(output_file_path, "w") as f: - yaml.dump(current_toc, f) - - -class TocSection: - def __init__(self, dir_name, index_file_name): - """ - :param dir_name: name of the directory containing the rst files - :param index_file_name: name of an index file within dir_name. This file - will not be included in the table of contents, but provides an ordered - list of the other files which should be included - """ - self.dir_name = dir_name - self.index_file_name = index_file_name - index_file_path = os.path.join(dir_name, index_file_name) - # find set of files referenced by the index file - with open(index_file_path, "r") as f: - self.title = None - in_toc = False - self.items = [] - for line in f: - # ignore empty lines - if not line.strip(): - continue - # add files explictly included in the toc - if line.startswith(".. include::"): - file_base = os.path.splitext(line.split("::")[1].strip())[0] - self.items.append( - self.extract_toc_entry( - file_base, file_title=file_base.capitalize() - ) - ) - continue - if line.startswith(".. toctree::"): - in_toc = True - continue - # ignore directives - if ":" in line: - continue - # set tile as first line with no directive - if self.title is None: - self.title = line.strip() - if not in_toc: - continue - # bail when toc indented block is done - if not line.startswith(" ") and not line.startswith("\t"): - in_toc = False - continue - # extract entries - self.items.append(self.extract_toc_entry(line.strip())) - - def extract_toc_entry(self, file_name, file_title=None): - """ - Given the name of a file, extract the title and href for the toc entry, - and return as a dictionary - """ - # load the file to get the title - with open(f"{self.dir_name}/{file_name}.rst", "r") as f2: - if file_title is None: - # use first line as title if not provided - file_title = f2.readline().strip() - return {"name": file_title, "href": f"{file_name}.md"} - - def to_dict(self): - """ - Convert the TocSection object to a dictionary that can be written to a yaml file - """ - return {"name": self.title, "items": self.items} - - def copy_markdown(self): - """ - Copy markdown files from _build/markdown/dir_name to _build/html/docfx_yaml - - This is necessary because the markdown files in sub-directories - are not copied over by the docfx build by default - """ - for file in os.listdir("_build/markdown/" + self.dir_name): - shutil.copy( - f"_build/markdown/{self.dir_name}/{file}", - f"_build/html/docfx_yaml", - ) - - -def validate_toc(toc_file_path, expected_section_list, added_sections): - current_toc = yaml.safe_load(open(toc_file_path, "r")) - # make sure the set of sections matches what we expect - found_sections = [d["name"] for d in current_toc[0]["items"]] - assert ( - found_sections == expected_section_list - ), f"Expected {expected_section_list}, found {found_sections}" - # make sure each customs ection is in the toc - for section in added_sections: - assert section.title in found_sections - # make sure each rst file in each custom section dir is listed in the toc - for section in added_sections: - items_in_toc = [ - d["items"] - for d in current_toc[0]["items"] - if d["name"] == section.title and ".rst" - ][0] - items_in_dir = [f for f in os.listdir(section.dir_name) if f.endswith(".rst")] - # subtract 1 for index - assert len(items_in_toc) == len(items_in_dir) - 1 - for file in items_in_dir: - if file != section.index_file_name: - base_name, _ = os.path.splitext(file) - assert any(d["href"] == f"{base_name}.md" for d in items_in_toc) - # make sure the markdown files are present in the docfx_yaml directory - for section in added_sections: - items_in_toc = [ - d["items"] - for d in current_toc[0]["items"] - if d["name"] == section.title and ".rst" - ][0] - md_files = [d["href"] for d in items_in_toc] - for file in md_files: - assert os.path.exists(f"_build/html/docfx_yaml/{file}") - print("Toc validation passed") - - -if __name__ == "__main__": - # Add secrtions for the async_data_client and classic_client directories - toc_path = "_build/html/docfx_yaml/toc.yml" - custom_sections = [ - TocSection(dir_name="data_client", index_file_name="data_client_usage.rst"), - TocSection(dir_name="classic_client", index_file_name="usage.rst"), - ] - add_sections(toc_path, custom_sections) - # Remove the Bigtable section, since it has duplicated data - remove_sections(toc_path, ["Bigtable"]) - # run validation to make sure yaml is structured as we expect - validate_toc( - toc_file_path=toc_path, - expected_section_list=[ - "Overview", - "bigtable APIs", - "Changelog", - "Multiprocessing", - "Data Client", - "Classic Client", - ], - added_sections=custom_sections, - ) diff --git a/noxfile.py b/noxfile.py index 548bfd0ec..a2142b1ae 100644 --- a/noxfile.py +++ b/noxfile.py @@ -451,9 +451,6 @@ def docfx(session): os.path.join("docs", ""), os.path.join("docs", "_build", "html", ""), ) - # Customization: Add extra sections to the table of contents for the Classic vs Async clients - session.install("pyyaml") - session.run("python", "docs/scripts/patch_devsite_toc.py") @nox.session(python="3.12") From 04b9e3730286f3fa4f6caacb84e63874a908f224 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 7 Apr 2025 13:17:57 -0700 Subject: [PATCH 2/6] removed classic client docs --- docs/classic_client/app-profile.rst | 6 - docs/classic_client/backup.rst | 6 - docs/classic_client/batcher.rst | 6 - docs/classic_client/client-intro.rst | 90 -- docs/classic_client/client.rst | 6 - docs/classic_client/cluster.rst | 6 - docs/classic_client/column-family.rst | 49 - docs/classic_client/data-api.rst | 351 ------ docs/classic_client/encryption-info.rst | 6 - docs/classic_client/instance-api.rst | 130 --- docs/classic_client/instance.rst | 6 - docs/classic_client/row-data.rst | 6 - docs/classic_client/row-filters.rst | 67 -- docs/classic_client/row-set.rst | 6 - docs/classic_client/row.rst | 7 - docs/classic_client/snippets.py | 793 -------------- docs/classic_client/snippets_table.py | 1340 ----------------------- docs/classic_client/table-api.rst | 153 --- docs/classic_client/table.rst | 6 - docs/classic_client/usage.rst | 38 - docs/index.rst | 1 - 21 files changed, 3079 deletions(-) delete mode 100644 docs/classic_client/app-profile.rst delete mode 100644 docs/classic_client/backup.rst delete mode 100644 docs/classic_client/batcher.rst delete mode 100644 docs/classic_client/client-intro.rst delete mode 100644 docs/classic_client/client.rst delete mode 100644 docs/classic_client/cluster.rst delete mode 100644 docs/classic_client/column-family.rst delete mode 100644 docs/classic_client/data-api.rst delete mode 100644 docs/classic_client/encryption-info.rst delete mode 100644 docs/classic_client/instance-api.rst delete mode 100644 docs/classic_client/instance.rst delete mode 100644 docs/classic_client/row-data.rst delete mode 100644 docs/classic_client/row-filters.rst delete mode 100644 docs/classic_client/row-set.rst delete mode 100644 docs/classic_client/row.rst delete mode 100644 docs/classic_client/snippets.py delete mode 100644 docs/classic_client/snippets_table.py delete mode 100644 docs/classic_client/table-api.rst delete mode 100644 docs/classic_client/table.rst delete mode 100644 docs/classic_client/usage.rst diff --git a/docs/classic_client/app-profile.rst b/docs/classic_client/app-profile.rst deleted file mode 100644 index 5c9d426c2..000000000 --- a/docs/classic_client/app-profile.rst +++ /dev/null @@ -1,6 +0,0 @@ -App Profile -~~~~~~~~~~~ - -.. automodule:: google.cloud.bigtable.app_profile - :members: - :show-inheritance: diff --git a/docs/classic_client/backup.rst b/docs/classic_client/backup.rst deleted file mode 100644 index e75abd431..000000000 --- a/docs/classic_client/backup.rst +++ /dev/null @@ -1,6 +0,0 @@ -Backup -~~~~~~~~ - -.. automodule:: google.cloud.bigtable.backup - :members: - :show-inheritance: diff --git a/docs/classic_client/batcher.rst b/docs/classic_client/batcher.rst deleted file mode 100644 index 9ac335be1..000000000 --- a/docs/classic_client/batcher.rst +++ /dev/null @@ -1,6 +0,0 @@ -Mutations Batching -~~~~~~~~~~~~~~~~~~ - -.. automodule:: google.cloud.bigtable.batcher - :members: - :show-inheritance: diff --git a/docs/classic_client/client-intro.rst b/docs/classic_client/client-intro.rst deleted file mode 100644 index 242068499..000000000 --- a/docs/classic_client/client-intro.rst +++ /dev/null @@ -1,90 +0,0 @@ -Base for Everything -=================== - -To use the API, the :class:`Client ` -class defines a high-level interface which handles authorization -and creating other objects: - -.. code:: python - - from google.cloud.bigtable.client import Client - client = Client() - -Long-lived Defaults -------------------- - -When creating a :class:`Client `, the -``user_agent`` argument has sensible a default -(:data:`DEFAULT_USER_AGENT `). -However, you may over-ride it and the value will be used throughout all API -requests made with the ``client`` you create. - -Configuration -------------- - -- For an overview of authentication in ``google-cloud-python``, - see `Authentication `_. - -- In addition to any authentication configuration, you can also set the - :envvar:`GOOGLE_CLOUD_PROJECT` environment variable for the Google Cloud Console - project you'd like to interact with. If your code is running in Google App - Engine or Google Compute Engine the project will be detected automatically. - (Setting this environment variable is not required, you may instead pass the - ``project`` explicitly when constructing a - :class:`Client `). - -- After configuring your environment, create a - :class:`Client ` - - .. code:: - - >>> from google.cloud import bigtable - >>> client = bigtable.Client() - - or pass in ``credentials`` and ``project`` explicitly - - .. code:: - - >>> from google.cloud import bigtable - >>> client = bigtable.Client(project='my-project', credentials=creds) - -.. tip:: - - Be sure to use the **Project ID**, not the **Project Number**. - -Admin API Access ----------------- - -If you'll be using your client to make `Instance Admin`_ and `Table Admin`_ -API requests, you'll need to pass the ``admin`` argument: - -.. code:: python - - client = bigtable.Client(admin=True) - -Read-Only Mode --------------- - -If, on the other hand, you only have (or want) read access to the data, -you can pass the ``read_only`` argument: - -.. code:: python - - client = bigtable.Client(read_only=True) - -This will ensure that the -:data:`READ_ONLY_SCOPE ` is used -for API requests (so any accidental requests that would modify data will -fail). - -Next Step ---------- - -After a :class:`Client `, the next highest-level -object is an :class:`Instance `. You'll need -one before you can interact with tables or data. - -Head next to learn about the :doc:`instance-api`. - -.. _Instance Admin: https://github.com/googleapis/python-bigtable/blob/main/google/cloud/bigtable_admin_v2/proto/bigtable_instance_admin.proto -.. _Table Admin: https://github.com/googleapis/python-bigtable/blob/main/google/cloud/bigtable_admin_v2/proto/bigtable_table_admin.proto diff --git a/docs/classic_client/client.rst b/docs/classic_client/client.rst deleted file mode 100644 index c48595c8a..000000000 --- a/docs/classic_client/client.rst +++ /dev/null @@ -1,6 +0,0 @@ -Client -~~~~~~ - -.. automodule:: google.cloud.bigtable.client - :members: - :show-inheritance: diff --git a/docs/classic_client/cluster.rst b/docs/classic_client/cluster.rst deleted file mode 100644 index ad33aae5e..000000000 --- a/docs/classic_client/cluster.rst +++ /dev/null @@ -1,6 +0,0 @@ -Cluster -~~~~~~~ - -.. automodule:: google.cloud.bigtable.cluster - :members: - :show-inheritance: diff --git a/docs/classic_client/column-family.rst b/docs/classic_client/column-family.rst deleted file mode 100644 index de6c1eb1f..000000000 --- a/docs/classic_client/column-family.rst +++ /dev/null @@ -1,49 +0,0 @@ -Column Families -=============== - -When creating a -:class:`ColumnFamily `, it is -possible to set garbage collection rules for expired data. - -By setting a rule, cells in the table matching the rule will be deleted -during periodic garbage collection (which executes opportunistically in the -background). - -The types -:class:`MaxAgeGCRule `, -:class:`MaxVersionsGCRule `, -:class:`GarbageCollectionRuleUnion ` and -:class:`GarbageCollectionRuleIntersection ` -can all be used as the optional ``gc_rule`` argument in the -:class:`ColumnFamily ` -constructor. This value is then used in the -:meth:`create() ` and -:meth:`update() ` methods. - -These rules can be nested arbitrarily, with a -:class:`MaxAgeGCRule ` or -:class:`MaxVersionsGCRule ` -at the lowest level of the nesting: - -.. code:: python - - import datetime - - max_age = datetime.timedelta(days=3) - rule1 = MaxAgeGCRule(max_age) - rule2 = MaxVersionsGCRule(1) - - # Make a composite that matches anything older than 3 days **AND** - # with more than 1 version. - rule3 = GarbageCollectionIntersection(rules=[rule1, rule2]) - - # Make another composite that matches our previous intersection - # **OR** anything that has more than 3 versions. - rule4 = GarbageCollectionRule(max_num_versions=3) - rule5 = GarbageCollectionUnion(rules=[rule3, rule4]) - ----- - -.. automodule:: google.cloud.bigtable.column_family - :members: - :show-inheritance: diff --git a/docs/classic_client/data-api.rst b/docs/classic_client/data-api.rst deleted file mode 100644 index 9b50e9ec9..000000000 --- a/docs/classic_client/data-api.rst +++ /dev/null @@ -1,351 +0,0 @@ -Data API -======== - -.. note:: - This page describes how to use the Data API with the synchronous Bigtable client. - Examples for using the Data API with the async client can be found in the - `Getting Started Guide`_. - -.. _Getting Started Guide: https://cloud.google.com/bigtable/docs/samples-python-hello - -After creating a :class:`Table ` and some -column families, you are ready to store and retrieve data. - -Cells vs. Columns vs. Column Families -+++++++++++++++++++++++++++++++++++++ - -* As explained in the :doc:`table overview `, tables can - have many column families. -* As described below, a table can also have many rows which are - specified by row keys. -* Within a row, data is stored in a cell. A cell simply has a value (as - bytes) and a timestamp. The number of cells in each row can be - different, depending on what was stored in each row. -* Each cell lies in a column (**not** a column family). A column is really - just a more **specific** modifier within a column family. A column - can be present in every column family, in only one or anywhere in between. -* Within a column family there can be many columns. For example, within - the column family ``foo`` we could have columns ``bar`` and ``baz``. - These would typically be represented as ``foo:bar`` and ``foo:baz``. - -Modifying Data -++++++++++++++ - -Since data is stored in cells, which are stored in rows, we -use the metaphor of a **row** in classes that are used to modify -(write, update, delete) data in a -:class:`Table `. - -Direct vs. Conditional vs. Append ---------------------------------- - -There are three ways to modify data in a table, described by the -`MutateRow`_, `CheckAndMutateRow`_ and `ReadModifyWriteRow`_ API -methods. - -* The **direct** way is via `MutateRow`_ which involves simply - adding, overwriting or deleting cells. The - :class:`DirectRow ` class - handles direct mutations. -* The **conditional** way is via `CheckAndMutateRow`_. This method - first checks if some filter is matched in a given row, then - applies one of two sets of mutations, depending on if a match - occurred or not. (These mutation sets are called the "true - mutations" and "false mutations".) The - :class:`ConditionalRow ` class - handles conditional mutations. -* The **append** way is via `ReadModifyWriteRow`_. This simply - appends (as bytes) or increments (as an integer) data in a presumed - existing cell in a row. The - :class:`AppendRow ` class - handles append mutations. - -Row Factory ------------ - -A single factory can be used to create any of the three row types. -To create a :class:`DirectRow `: - -.. code:: python - - row = table.row(row_key) - -Unlike the previous string values we've used before, the row key must -be ``bytes``. - -To create a :class:`ConditionalRow `, -first create a :class:`RowFilter ` and -then - -.. code:: python - - cond_row = table.row(row_key, filter_=filter_) - -To create an :class:`AppendRow ` - -.. code:: python - - append_row = table.row(row_key, append=True) - -Building Up Mutations ---------------------- - -In all three cases, a set of mutations (or two sets) are built up -on a row before they are sent off in a batch via - -.. code:: python - - row.commit() - -Direct Mutations ----------------- - -Direct mutations can be added via one of four methods - -* :meth:`set_cell() ` allows a - single value to be written to a column - - .. code:: python - - row.set_cell(column_family_id, column, value, - timestamp=timestamp) - - If the ``timestamp`` is omitted, the current time on the Google Cloud - Bigtable server will be used when the cell is stored. - - The value can either be bytes or an integer, which will be converted to - bytes as a signed 64-bit integer. - -* :meth:`delete_cell() ` deletes - all cells (i.e. for all timestamps) in a given column - - .. code:: python - - row.delete_cell(column_family_id, column) - - Remember, this only happens in the ``row`` we are using. - - If we only want to delete cells from a limited range of time, a - :class:`TimestampRange ` can - be used - - .. code:: python - - row.delete_cell(column_family_id, column, - time_range=time_range) - -* :meth:`delete_cells() ` does - the same thing as - :meth:`delete_cell() `, - but accepts a list of columns in a column family rather than a single one. - - .. code:: python - - row.delete_cells(column_family_id, [column1, column2], - time_range=time_range) - - In addition, if we want to delete cells from every column in a column family, - the special :attr:`ALL_COLUMNS ` - value can be used - - .. code:: python - - row.delete_cells(column_family_id, row.ALL_COLUMNS, - time_range=time_range) - -* :meth:`delete() ` will delete the - entire row - - .. code:: python - - row.delete() - -Conditional Mutations ---------------------- - -Making **conditional** modifications is essentially identical -to **direct** modifications: it uses the exact same methods -to accumulate mutations. - -However, each mutation added must specify a ``state``: will the mutation be -applied if the filter matches or if it fails to match. - -For example: - -.. code:: python - - cond_row.set_cell(column_family_id, column, value, - timestamp=timestamp, state=True) - -will add to the set of true mutations. - -Append Mutations ----------------- - -Append mutations can be added via one of two methods - -* :meth:`append_cell_value() ` - appends a bytes value to an existing cell: - - .. code:: python - - append_row.append_cell_value(column_family_id, column, bytes_value) - -* :meth:`increment_cell_value() ` - increments an integer value in an existing cell: - - .. code:: python - - append_row.increment_cell_value(column_family_id, column, int_value) - - Since only bytes are stored in a cell, the cell value is decoded as - a signed 64-bit integer before being incremented. (This happens on - the Google Cloud Bigtable server, not in the library.) - -Notice that no timestamp was specified. This is because **append** mutations -operate on the latest value of the specified column. - -If there are no cells in the specified column, then the empty string (bytes -case) or zero (integer case) are the assumed values. - -Starting Fresh --------------- - -If accumulated mutations need to be dropped, use - -.. code:: python - - row.clear() - -Reading Data -++++++++++++ - -Read Single Row from a Table ----------------------------- - -To make a `ReadRows`_ API request for a single row key, use -:meth:`Table.read_row() `: - -.. code:: python - - >>> row_data = table.read_row(row_key) - >>> row_data.cells - { - u'fam1': { - b'col1': [ - , - , - ], - b'col2': [ - , - ], - }, - u'fam2': { - b'col3': [ - , - , - , - ], - }, - } - >>> cell = row_data.cells[u'fam1'][b'col1'][0] - >>> cell - - >>> cell.value - b'val1' - >>> cell.timestamp - datetime.datetime(2016, 2, 27, 3, 41, 18, 122823, tzinfo=) - -Rather than returning a :class:`DirectRow ` -or similar class, this method returns a -:class:`PartialRowData ` -instance. This class is used for reading and parsing data rather than for -modifying data (as :class:`DirectRow ` is). - -A filter can also be applied to the results: - -.. code:: python - - row_data = table.read_row(row_key, filter_=filter_val) - -The allowable ``filter_`` values are the same as those used for a -:class:`ConditionalRow `. For -more information, see the -:meth:`Table.read_row() ` documentation. - -Stream Many Rows from a Table ------------------------------ - -To make a `ReadRows`_ API request for a stream of rows, use -:meth:`Table.read_rows() `: - -.. code:: python - - row_data = table.read_rows() - -Using gRPC over HTTP/2, a continual stream of responses will be delivered. -In particular - -* :meth:`consume_next() ` - pulls the next result from the stream, parses it and stores it on the - :class:`PartialRowsData ` instance -* :meth:`consume_all() ` - pulls results from the stream until there are no more -* :meth:`cancel() ` closes - the stream - -See the :class:`PartialRowsData ` -documentation for more information. - -As with -:meth:`Table.read_row() `, an optional -``filter_`` can be applied. In addition a ``start_key`` and / or ``end_key`` -can be supplied for the stream, a ``limit`` can be set and a boolean -``allow_row_interleaving`` can be specified to allow faster streamed results -at the potential cost of non-sequential reads. - -See the :meth:`Table.read_rows() ` -documentation for more information on the optional arguments. - -Sample Keys in a Table ----------------------- - -Make a `SampleRowKeys`_ API request with -:meth:`Table.sample_row_keys() `: - -.. code:: python - - keys_iterator = table.sample_row_keys() - -The returned row keys will delimit contiguous sections of the table of -approximately equal size, which can be used to break up the data for -distributed tasks like mapreduces. - -As with -:meth:`Table.read_rows() `, the -returned ``keys_iterator`` is connected to a cancellable HTTP/2 stream. - -The next key in the result can be accessed via - -.. code:: python - - next_key = keys_iterator.next() - -or all keys can be iterated over via - -.. code:: python - - for curr_key in keys_iterator: - do_something(curr_key) - -Just as with reading, the stream can be canceled: - -.. code:: python - - keys_iterator.cancel() - -.. _ReadRows: https://github.com/googleapis/python-bigtable/blob/d6bff70654b41e31d2ac83d307bdc6bbd111201e/google/cloud/bigtable_v2/types/bigtable.py#L42-L72 -.. _SampleRowKeys: https://github.com/googleapis/python-bigtable/blob/d6bff70654b41e31d2ac83d307bdc6bbd111201e/google/cloud/bigtable_v2/types/bigtable.py#L184-L199 -.. _MutateRow: https://github.com/googleapis/python-bigtable/blob/d6bff70654b41e31d2ac83d307bdc6bbd111201e/google/cloud/bigtable_v2/types/bigtable.py#L230-L256 -.. _CheckAndMutateRow: https://github.com/googleapis/python-bigtable/blob/d6bff70654b41e31d2ac83d307bdc6bbd111201e/google/cloud/bigtable_v2/types/bigtable.py#L339-L386 -.. _ReadModifyWriteRow: https://github.com/googleapis/python-bigtable/blob/d6bff70654b41e31d2ac83d307bdc6bbd111201e/google/cloud/bigtable_v2/types/bigtable.py#L401-L430 diff --git a/docs/classic_client/encryption-info.rst b/docs/classic_client/encryption-info.rst deleted file mode 100644 index 46f19880f..000000000 --- a/docs/classic_client/encryption-info.rst +++ /dev/null @@ -1,6 +0,0 @@ -Encryption Info -~~~~~~~~~~~~~~~ - -.. automodule:: google.cloud.bigtable.encryption_info - :members: - :show-inheritance: diff --git a/docs/classic_client/instance-api.rst b/docs/classic_client/instance-api.rst deleted file mode 100644 index 88b4eb4dc..000000000 --- a/docs/classic_client/instance-api.rst +++ /dev/null @@ -1,130 +0,0 @@ -Instance Admin API -================== - -After creating a :class:`Client `, you can -interact with individual instances for a project. - -List Instances --------------- - -If you want a comprehensive list of all existing instances, make a -`ListInstances`_ API request with -:meth:`Client.list_instances() `: - -.. code:: python - - instances = client.list_instances() - -Instance Factory ----------------- - -To create an :class:`Instance ` object: - -.. code:: python - - instance = client.instance(instance_id, display_name=display_name) - -- ``display_name`` is optional. When not provided, ``display_name`` defaults - to the ``instance_id`` value. - -You can also use :meth:`Client.instance` to create a local wrapper for -instances that have already been created with the API, or through the web -console: - -.. code:: python - - instance = client.instance(existing_instance_id) - instance.reload() - -Create a new Instance ---------------------- - -After creating the instance object, make a `CreateInstance`_ API request -with :meth:`create() `: - -.. code:: python - - instance.display_name = 'My very own instance' - instance.create() - -Check on Current Operation --------------------------- - -.. note:: - - When modifying an instance (via a `CreateInstance`_ request), the Bigtable - API will return a `long-running operation`_ and a corresponding - :class:`Operation ` object - will be returned by - :meth:`create() `. - -You can check if a long-running operation (for a -:meth:`create() ` has finished -by making a `GetOperation`_ request with -:meth:`Operation.finished() `: - -.. code:: python - - >>> operation = instance.create() - >>> operation.finished() - True - -.. note:: - - Once an :class:`Operation ` object - has returned :data:`True` from - :meth:`finished() `, the - object should not be re-used. Subsequent calls to - :meth:`finished() ` - will result in a :class:`ValueError `. - -Get metadata for an existing Instance -------------------------------------- - -After creating the instance object, make a `GetInstance`_ API request -with :meth:`reload() `: - -.. code:: python - - instance.reload() - -This will load ``display_name`` for the existing ``instance`` object. - -Update an existing Instance ---------------------------- - -After creating the instance object, make an `UpdateInstance`_ API request -with :meth:`update() `: - -.. code:: python - - instance.display_name = 'New display_name' - instance.update() - -Delete an existing Instance ---------------------------- - -Make a `DeleteInstance`_ API request with -:meth:`delete() `: - -.. code:: python - - instance.delete() - -Next Step ---------- - -Now we go down the hierarchy from -:class:`Instance ` to a -:class:`Table `. - -Head next to learn about the :doc:`table-api`. - -.. _Instance Admin API: https://cloud.google.com/bigtable/docs/creating-instance -.. _CreateInstance: https://googleapis.dev/python/bigtable/latest/instance-api.html#create-a-new-instance -.. _GetInstance: https://googleapis.dev/python/bigtable/latest/instance-api.html#get-metadata-for-an-existing-instance -.. _UpdateInstance: https://googleapis.dev/python/bigtable/latest/instance-api.html#update-an-existing-instance -.. _DeleteInstance: https://googleapis.dev/python/bigtable/latest/instance-api.html#delete-an-existing-instance -.. _ListInstances: https://googleapis.dev/python/bigtable/latest/instance-api.html#list-instances -.. _GetOperation: https://googleapis.dev/python/bigtable/latest/instance-api.html#check-on-current-operation -.. _long-running operation: https://github.com/googleapis/googleapis/blob/main/google/longrunning/operations.proto#L128-L162 diff --git a/docs/classic_client/instance.rst b/docs/classic_client/instance.rst deleted file mode 100644 index f9be9672f..000000000 --- a/docs/classic_client/instance.rst +++ /dev/null @@ -1,6 +0,0 @@ -Instance -~~~~~~~~ - -.. automodule:: google.cloud.bigtable.instance - :members: - :show-inheritance: diff --git a/docs/classic_client/row-data.rst b/docs/classic_client/row-data.rst deleted file mode 100644 index 503f9b1cb..000000000 --- a/docs/classic_client/row-data.rst +++ /dev/null @@ -1,6 +0,0 @@ -Row Data -~~~~~~~~ - -.. automodule:: google.cloud.bigtable.row_data - :members: - :show-inheritance: diff --git a/docs/classic_client/row-filters.rst b/docs/classic_client/row-filters.rst deleted file mode 100644 index 9884ce400..000000000 --- a/docs/classic_client/row-filters.rst +++ /dev/null @@ -1,67 +0,0 @@ -Bigtable Row Filters -==================== - -It is possible to use a -:class:`RowFilter ` -when adding mutations to a -:class:`ConditionalRow ` and when -reading row data with :meth:`read_row() ` -or :meth:`read_rows() `. - -As laid out in the `RowFilter definition`_, the following basic filters -are provided: - -* :class:`SinkFilter <.row_filters.SinkFilter>` -* :class:`PassAllFilter <.row_filters.PassAllFilter>` -* :class:`BlockAllFilter <.row_filters.BlockAllFilter>` -* :class:`RowKeyRegexFilter <.row_filters.RowKeyRegexFilter>` -* :class:`RowSampleFilter <.row_filters.RowSampleFilter>` -* :class:`FamilyNameRegexFilter <.row_filters.FamilyNameRegexFilter>` -* :class:`ColumnQualifierRegexFilter <.row_filters.ColumnQualifierRegexFilter>` -* :class:`TimestampRangeFilter <.row_filters.TimestampRangeFilter>` -* :class:`ColumnRangeFilter <.row_filters.ColumnRangeFilter>` -* :class:`ValueRegexFilter <.row_filters.ValueRegexFilter>` -* :class:`ValueRangeFilter <.row_filters.ValueRangeFilter>` -* :class:`CellsRowOffsetFilter <.row_filters.CellsRowOffsetFilter>` -* :class:`CellsRowLimitFilter <.row_filters.CellsRowLimitFilter>` -* :class:`CellsColumnLimitFilter <.row_filters.CellsColumnLimitFilter>` -* :class:`StripValueTransformerFilter <.row_filters.StripValueTransformerFilter>` -* :class:`ApplyLabelFilter <.row_filters.ApplyLabelFilter>` - -In addition, these filters can be combined into composite filters with - -* :class:`RowFilterChain <.row_filters.RowFilterChain>` -* :class:`RowFilterUnion <.row_filters.RowFilterUnion>` -* :class:`ConditionalRowFilter <.row_filters.ConditionalRowFilter>` - -These rules can be nested arbitrarily, with a basic filter at the lowest -level. For example: - -.. code:: python - - # Filter in a specified column (matching any column family). - col1_filter = ColumnQualifierRegexFilter(b'columnbia') - - # Create a filter to label results. - label1 = u'label-red' - label1_filter = ApplyLabelFilter(label1) - - # Combine the filters to label all the cells in columnbia. - chain1 = RowFilterChain(filters=[col1_filter, label1_filter]) - - # Create a similar filter to label cells blue. - col2_filter = ColumnQualifierRegexFilter(b'columnseeya') - label2 = u'label-blue' - label2_filter = ApplyLabelFilter(label2) - chain2 = RowFilterChain(filters=[col2_filter, label2_filter]) - - # Bring our two labeled columns together. - row_filter = RowFilterUnion(filters=[chain1, chain2]) - ----- - -.. automodule:: google.cloud.bigtable.row_filters - :members: - :show-inheritance: - -.. _RowFilter definition: https://googleapis.dev/python/bigtable/latest/row-filters.html?highlight=rowfilter#google.cloud.bigtable.row_filters.RowFilter diff --git a/docs/classic_client/row-set.rst b/docs/classic_client/row-set.rst deleted file mode 100644 index 5f7a16a02..000000000 --- a/docs/classic_client/row-set.rst +++ /dev/null @@ -1,6 +0,0 @@ -Row Set -~~~~~~~~ - -.. automodule:: google.cloud.bigtable.row_set - :members: - :show-inheritance: diff --git a/docs/classic_client/row.rst b/docs/classic_client/row.rst deleted file mode 100644 index 33686608b..000000000 --- a/docs/classic_client/row.rst +++ /dev/null @@ -1,7 +0,0 @@ -Bigtable Row -============ - -.. automodule:: google.cloud.bigtable.row - :members: - :show-inheritance: - :inherited-members: diff --git a/docs/classic_client/snippets.py b/docs/classic_client/snippets.py deleted file mode 100644 index fa3aa3627..000000000 --- a/docs/classic_client/snippets.py +++ /dev/null @@ -1,793 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018, Google LLC -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Testable usage examples for Google Cloud Bigtable API wrapper - -Each example function takes a ``client`` argument (which must be an instance -of :class:`google.cloud.bigtable.client.Client`) and uses it to perform a task -with the API. - -To facilitate running the examples as system tests, each example is also passed -a ``to_delete`` list; the function adds to the list any objects created which -need to be deleted during teardown. - -.. note:: - This file is under progress and will be updated with more guidance from - the team. Unit tests will be added with guidance from the team. - -""" - -import datetime -import pytest - -from google.api_core.exceptions import DeadlineExceeded -from google.api_core.exceptions import NotFound -from google.api_core.exceptions import TooManyRequests -from google.api_core.exceptions import ServiceUnavailable -from test_utils.system import unique_resource_id -from test_utils.retry import RetryErrors - -from google.cloud._helpers import UTC -from google.cloud.bigtable import Client -from google.cloud.bigtable import enums - - -UNIQUE_SUFFIX = unique_resource_id("-") -INSTANCE_ID = "snippet-tests" + UNIQUE_SUFFIX -CLUSTER_ID = "clus-1-" + UNIQUE_SUFFIX -APP_PROFILE_ID = "app-prof" + UNIQUE_SUFFIX -TABLE_ID = "tabl-1" + UNIQUE_SUFFIX -ROUTING_POLICY_TYPE = enums.RoutingPolicyType.ANY -LOCATION_ID = "us-central1-f" -ALT_LOCATION_ID = "us-central1-a" -PRODUCTION = enums.Instance.Type.PRODUCTION -SERVER_NODES = 3 -STORAGE_TYPE = enums.StorageType.SSD -LABEL_KEY = "python-snippet" -LABEL_STAMP = ( - datetime.datetime.utcnow() - .replace(microsecond=0, tzinfo=UTC) - .strftime("%Y-%m-%dt%H-%M-%S") -) -LABELS = {LABEL_KEY: str(LABEL_STAMP)} -INSTANCES_TO_DELETE = [] - -retry_429_503 = RetryErrors((ServiceUnavailable, TooManyRequests), max_tries=9) -retry_504 = RetryErrors(DeadlineExceeded, max_tries=4) - - -class Config(object): - """Run-time configuration to be modified at set-up. - - This is a mutable stand-in to allow test set-up to modify - global state. - """ - - CLIENT = None - INSTANCE = None - TABLE = None - - -def setup_module(): - client = Config.CLIENT = Client(admin=True) - Config.INSTANCE = client.instance( - INSTANCE_ID, instance_type=PRODUCTION, labels=LABELS - ) - cluster = Config.INSTANCE.cluster( - CLUSTER_ID, - location_id=LOCATION_ID, - serve_nodes=SERVER_NODES, - default_storage_type=STORAGE_TYPE, - ) - operation = Config.INSTANCE.create(clusters=[cluster]) - # We want to make sure the operation completes. - operation.result(timeout=100) - Config.TABLE = Config.INSTANCE.table(TABLE_ID) - retry_504(Config.TABLE.create)() - - -def teardown_module(): - retry_429_503(Config.INSTANCE.delete)() - - for instance in INSTANCES_TO_DELETE: - try: - retry_429_503(instance.delete)() - except NotFound: - pass - - -def test_bigtable_create_instance(): - # [START bigtable_api_create_prod_instance] - from google.cloud.bigtable import Client - from google.cloud.bigtable import enums - - my_instance_id = "inst-my-" + UNIQUE_SUFFIX - my_cluster_id = "clus-my-" + UNIQUE_SUFFIX - location_id = "us-central1-f" - serve_nodes = 1 - storage_type = enums.StorageType.SSD - production = enums.Instance.Type.PRODUCTION - labels = {"prod-label": "prod-label"} - - client = Client(admin=True) - instance = client.instance(my_instance_id, instance_type=production, labels=labels) - cluster = instance.cluster( - my_cluster_id, - location_id=location_id, - serve_nodes=serve_nodes, - default_storage_type=storage_type, - ) - operation = instance.create(clusters=[cluster]) - - # We want to make sure the operation completes. - operation.result(timeout=100) - - # [END bigtable_api_create_prod_instance] - - try: - assert instance.exists() - finally: - retry_429_503(instance.delete)() - - -def test_bigtable_create_additional_cluster(): - # [START bigtable_api_create_cluster] - from google.cloud.bigtable import Client - from google.cloud.bigtable import enums - - # Assuming that there is an existing instance with `INSTANCE_ID` - # on the server already. - # to create an instance see - # 'https://cloud.google.com/bigtable/docs/creating-instance' - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - - cluster_id = "clus-my-" + UNIQUE_SUFFIX - location_id = "us-central1-a" - serve_nodes = 1 - storage_type = enums.StorageType.SSD - - cluster = instance.cluster( - cluster_id, - location_id=location_id, - serve_nodes=serve_nodes, - default_storage_type=storage_type, - ) - operation = cluster.create() - # We want to make sure the operation completes. - operation.result(timeout=100) - # [END bigtable_api_create_cluster] - - try: - assert cluster.exists() - finally: - retry_429_503(cluster.delete)() - - -def test_bigtable_create_reload_delete_app_profile(): - import re - - # [START bigtable_api_create_app_profile] - from google.cloud.bigtable import Client - from google.cloud.bigtable import enums - - routing_policy_type = enums.RoutingPolicyType.ANY - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - - description = "routing policy-multy" - - app_profile = instance.app_profile( - app_profile_id=APP_PROFILE_ID, - routing_policy_type=routing_policy_type, - description=description, - cluster_id=CLUSTER_ID, - ) - - app_profile = app_profile.create(ignore_warnings=True) - # [END bigtable_api_create_app_profile] - - # [START bigtable_api_app_profile_name] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - app_profile = instance.app_profile(APP_PROFILE_ID) - - app_profile_name = app_profile.name - # [END bigtable_api_app_profile_name] - _profile_name_re = re.compile( - r"^projects/(?P[^/]+)/" - r"instances/(?P[^/]+)/" - r"appProfiles/(?P" - r"[_a-zA-Z0-9][-_.a-zA-Z0-9]*)$" - ) - assert _profile_name_re.match(app_profile_name) - - # [START bigtable_api_app_profile_exists] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - app_profile = instance.app_profile(APP_PROFILE_ID) - - app_profile_exists = app_profile.exists() - # [END bigtable_api_app_profile_exists] - assert app_profile_exists - - # [START bigtable_api_reload_app_profile] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - app_profile = instance.app_profile(APP_PROFILE_ID) - - app_profile.reload() - # [END bigtable_api_reload_app_profile] - assert app_profile.routing_policy_type == ROUTING_POLICY_TYPE - - # [START bigtable_api_update_app_profile] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - app_profile = instance.app_profile(APP_PROFILE_ID) - app_profile.reload() - - description = "My new app profile" - app_profile.description = description - app_profile.update() - # [END bigtable_api_update_app_profile] - assert app_profile.description == description - - # [START bigtable_api_delete_app_profile] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - app_profile = instance.app_profile(APP_PROFILE_ID) - app_profile.reload() - - app_profile.delete(ignore_warnings=True) - # [END bigtable_api_delete_app_profile] - assert not app_profile.exists() - - -def test_bigtable_list_instances(): - # [START bigtable_api_list_instances] - from google.cloud.bigtable import Client - - client = Client(admin=True) - (instances_list, failed_locations_list) = client.list_instances() - # [END bigtable_api_list_instances] - - assert len(instances_list) > 0 - - -def test_bigtable_list_clusters_on_instance(): - # [START bigtable_api_list_clusters_on_instance] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - (clusters_list, failed_locations_list) = instance.list_clusters() - # [END bigtable_api_list_clusters_on_instance] - - assert len(clusters_list) > 0 - - -def test_bigtable_list_clusters_in_project(): - # [START bigtable_api_list_clusters_in_project] - from google.cloud.bigtable import Client - - client = Client(admin=True) - (clusters_list, failed_locations_list) = client.list_clusters() - # [END bigtable_api_list_clusters_in_project] - - assert len(clusters_list) > 0 - - -def test_bigtable_list_app_profiles(): - app_profile = Config.INSTANCE.app_profile( - app_profile_id="app-prof-" + UNIQUE_SUFFIX, - routing_policy_type=enums.RoutingPolicyType.ANY, - ) - app_profile = app_profile.create(ignore_warnings=True) - - # [START bigtable_api_list_app_profiles] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - - app_profiles_list = instance.list_app_profiles() - # [END bigtable_api_list_app_profiles] - - try: - assert len(app_profiles_list) > 0 - finally: - retry_429_503(app_profile.delete)(ignore_warnings=True) - - -def test_bigtable_instance_exists(): - # [START bigtable_api_check_instance_exists] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance_exists = instance.exists() - # [END bigtable_api_check_instance_exists] - - assert instance_exists - - -def test_bigtable_cluster_exists(): - # [START bigtable_api_check_cluster_exists] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - cluster = instance.cluster(CLUSTER_ID) - cluster_exists = cluster.exists() - # [END bigtable_api_check_cluster_exists] - - assert cluster_exists - - -def test_bigtable_reload_instance(): - # [START bigtable_api_reload_instance] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance.reload() - # [END bigtable_api_reload_instance] - - assert instance.type_ == PRODUCTION.value - - -def test_bigtable_reload_cluster(): - # [START bigtable_api_reload_cluster] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - cluster = instance.cluster(CLUSTER_ID) - cluster.reload() - # [END bigtable_api_reload_cluster] - - assert cluster.serve_nodes == SERVER_NODES - - -def test_bigtable_update_instance(): - # [START bigtable_api_update_instance] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - display_name = "My new instance" - instance.display_name = display_name - instance.update() - # [END bigtable_api_update_instance] - - assert instance.display_name == display_name - - -def test_bigtable_update_cluster(): - # [START bigtable_api_update_cluster] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - cluster = instance.cluster(CLUSTER_ID) - cluster.serve_nodes = 4 - cluster.update() - # [END bigtable_api_update_cluster] - - assert cluster.serve_nodes == 4 - - -def test_bigtable_cluster_disable_autoscaling(): - # [START bigtable_api_cluster_disable_autoscaling] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - # Create a cluster with autoscaling enabled - cluster = instance.cluster( - CLUSTER_ID, min_serve_nodes=1, max_serve_nodes=2, cpu_utilization_percent=10 - ) - instance.create(clusters=[cluster]) - - # Disable autoscaling - cluster.disable_autoscaling(serve_nodes=4) - # [END bigtable_api_cluster_disable_autoscaling] - - assert cluster.serve_nodes == 4 - - -def test_bigtable_create_table(): - # [START bigtable_api_create_table] - from google.api_core import exceptions - from google.api_core import retry - from google.cloud.bigtable import Client - from google.cloud.bigtable import column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table("table_my") - # Define the GC policy to retain only the most recent 2 versions. - max_versions_rule = column_family.MaxVersionsGCRule(2) - - # Could include other retriable exception types - # Could configure deadline, etc. - predicate_504 = retry.if_exception_type(exceptions.DeadlineExceeded) - retry_504 = retry.Retry(predicate_504) - - retry_504(table.create)(column_families={"cf1": max_versions_rule}) - # [END bigtable_api_create_table] - - try: - assert table.exists() - finally: - retry_429_503(table.delete)() - - -def test_bigtable_list_tables(): - # [START bigtable_api_list_tables] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - tables_list = instance.list_tables() - # [END bigtable_api_list_tables] - - # Check if returned list has expected table - table_names = [table.name for table in tables_list] - assert Config.TABLE.name in table_names - - -def test_bigtable_delete_cluster(): - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - cluster_id = "clus-my-" + UNIQUE_SUFFIX - serve_nodes = 1 - cluster = instance.cluster( - cluster_id, - location_id=ALT_LOCATION_ID, - serve_nodes=serve_nodes, - default_storage_type=STORAGE_TYPE, - ) - operation = cluster.create() - # We want to make sure the operation completes. - operation.result(timeout=1000) - - # [START bigtable_api_delete_cluster] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - cluster_to_delete = instance.cluster(cluster_id) - - cluster_to_delete.delete() - # [END bigtable_api_delete_cluster] - - assert not cluster_to_delete.exists() - - -def test_bigtable_delete_instance(): - from google.cloud.bigtable import Client - - client = Client(admin=True) - - instance_id = "snipt-inst-del" + UNIQUE_SUFFIX - instance = client.instance(instance_id, instance_type=PRODUCTION, labels=LABELS) - serve_nodes = 1 - cluster = instance.cluster( - "clus-to-delete" + UNIQUE_SUFFIX, - location_id=ALT_LOCATION_ID, - serve_nodes=serve_nodes, - default_storage_type=STORAGE_TYPE, - ) - operation = instance.create(clusters=[cluster]) - - # We want to make sure the operation completes. - operation.result(timeout=100) - - # Make sure this instance gets deleted after the test case. - INSTANCES_TO_DELETE.append(instance) - - # [START bigtable_api_delete_instance] - from google.cloud.bigtable import Client - - client = Client(admin=True) - - instance_to_delete = client.instance(instance_id) - instance_to_delete.delete() - # [END bigtable_api_delete_instance] - - assert not instance_to_delete.exists() - - # Skip deleting it during module teardown if the assertion succeeds. - INSTANCES_TO_DELETE.remove(instance) - - -def test_bigtable_test_iam_permissions(): - # [START bigtable_api_test_iam_permissions] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance.reload() - permissions = ["bigtable.clusters.create", "bigtable.tables.create"] - permissions_allowed = instance.test_iam_permissions(permissions) - # [END bigtable_api_test_iam_permissions] - - assert permissions_allowed == permissions - - -def test_bigtable_set_iam_policy_then_get_iam_policy(): - service_account_email = Config.CLIENT._credentials.service_account_email - - # [START bigtable_api_set_iam_policy] - from google.cloud.bigtable import Client - from google.cloud.bigtable.policy import Policy - from google.cloud.bigtable.policy import BIGTABLE_ADMIN_ROLE - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance.reload() - new_policy = Policy() - new_policy[BIGTABLE_ADMIN_ROLE] = [Policy.service_account(service_account_email)] - - policy_latest = instance.set_iam_policy(new_policy) - # [END bigtable_api_set_iam_policy] - - assert len(policy_latest.bigtable_admins) > 0 - - # [START bigtable_api_get_iam_policy] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - policy = instance.get_iam_policy() - # [END bigtable_api_get_iam_policy] - - assert len(policy.bigtable_admins) > 0 - - -def test_bigtable_project_path(): - import re - - # [START bigtable_api_project_path] - from google.cloud.bigtable import Client - - client = Client(admin=True) - project_path = client.project_path - # [END bigtable_api_project_path] - - -def test_bigtable_table_data_client(): - # [START bigtable_api_table_data_client] - from google.cloud.bigtable import Client - - client = Client(admin=True) - table_data_client = client.table_data_client - # [END bigtable_api_table_data_client] - - -def test_bigtable_table_admin_client(): - # [START bigtable_api_table_admin_client] - from google.cloud.bigtable import Client - - client = Client(admin=True) - table_admin_client = client.table_admin_client - # [END bigtable_api_table_admin_client] - - -def test_bigtable_instance_admin_client(): - # [START bigtable_api_instance_admin_client] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance_admin_client = client.instance_admin_client - # [END bigtable_api_instance_admin_client] - - -def test_bigtable_admins_policy(): - service_account_email = Config.CLIENT._credentials.service_account_email - - # [START bigtable_api_admins_policy] - from google.cloud.bigtable import Client - from google.cloud.bigtable.policy import Policy - from google.cloud.bigtable.policy import BIGTABLE_ADMIN_ROLE - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance.reload() - new_policy = Policy() - new_policy[BIGTABLE_ADMIN_ROLE] = [Policy.service_account(service_account_email)] - - policy_latest = instance.set_iam_policy(new_policy) - policy = policy_latest.bigtable_admins - # [END bigtable_api_admins_policy] - - assert len(policy) > 0 - - -def test_bigtable_readers_policy(): - service_account_email = Config.CLIENT._credentials.service_account_email - - # [START bigtable_api_readers_policy] - from google.cloud.bigtable import Client - from google.cloud.bigtable.policy import Policy - from google.cloud.bigtable.policy import BIGTABLE_READER_ROLE - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance.reload() - new_policy = Policy() - new_policy[BIGTABLE_READER_ROLE] = [Policy.service_account(service_account_email)] - - policy_latest = instance.set_iam_policy(new_policy) - policy = policy_latest.bigtable_readers - # [END bigtable_api_readers_policy] - - assert len(policy) > 0 - - -def test_bigtable_users_policy(): - service_account_email = Config.CLIENT._credentials.service_account_email - - # [START bigtable_api_users_policy] - from google.cloud.bigtable import Client - from google.cloud.bigtable.policy import Policy - from google.cloud.bigtable.policy import BIGTABLE_USER_ROLE - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance.reload() - new_policy = Policy() - new_policy[BIGTABLE_USER_ROLE] = [Policy.service_account(service_account_email)] - - policy_latest = instance.set_iam_policy(new_policy) - policy = policy_latest.bigtable_users - # [END bigtable_api_users_policy] - - assert len(policy) > 0 - - -def test_bigtable_viewers_policy(): - service_account_email = Config.CLIENT._credentials.service_account_email - - # [START bigtable_api_viewers_policy] - from google.cloud.bigtable import Client - from google.cloud.bigtable.policy import Policy - from google.cloud.bigtable.policy import BIGTABLE_VIEWER_ROLE - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance.reload() - new_policy = Policy() - new_policy[BIGTABLE_VIEWER_ROLE] = [Policy.service_account(service_account_email)] - - policy_latest = instance.set_iam_policy(new_policy) - policy = policy_latest.bigtable_viewers - # [END bigtable_api_viewers_policy] - - assert len(policy) > 0 - - -def test_bigtable_instance_name(): - import re - - # [START bigtable_api_instance_name] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance_name = instance.name - # [END bigtable_api_instance_name] - - -def test_bigtable_cluster_name(): - import re - - # [START bigtable_api_cluster_name] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - cluster = instance.cluster(CLUSTER_ID) - cluster_name = cluster.name - # [END bigtable_api_cluster_name] - - -def test_bigtable_instance_from_pb(): - # [START bigtable_api_instance_from_pb] - from google.cloud.bigtable import Client - from google.cloud.bigtable_admin_v2.types import instance as data_v2_pb2 - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - - name = instance.name - instance_pb = data_v2_pb2.Instance( - name=name, display_name=INSTANCE_ID, type=PRODUCTION, labels=LABELS - ) - - instance2 = instance.from_pb(instance_pb, client) - # [END bigtable_api_instance_from_pb] - - assert instance2.name == instance.name - - -def test_bigtable_cluster_from_pb(): - # [START bigtable_api_cluster_from_pb] - from google.cloud.bigtable import Client - from google.cloud.bigtable_admin_v2.types import instance as data_v2_pb2 - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - cluster = instance.cluster(CLUSTER_ID) - - name = cluster.name - cluster_state = cluster.state - serve_nodes = 1 - cluster_pb = data_v2_pb2.Cluster( - name=name, - location=LOCATION_ID, - state=cluster_state, - serve_nodes=serve_nodes, - default_storage_type=STORAGE_TYPE, - ) - - cluster2 = cluster.from_pb(cluster_pb, instance) - # [END bigtable_api_cluster_from_pb] - - assert cluster2.name == cluster.name - - -def test_bigtable_instance_state(): - # [START bigtable_api_instance_state] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - instance_state = instance.state - # [END bigtable_api_instance_state] - - assert not instance_state - - -def test_bigtable_cluster_state(): - # [START bigtable_api_cluster_state] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - cluster = instance.cluster(CLUSTER_ID) - cluster_state = cluster.state - # [END bigtable_api_cluster_state] - - assert not cluster_state - - -if __name__ == "__main__": - pytest.main() diff --git a/docs/classic_client/snippets_table.py b/docs/classic_client/snippets_table.py deleted file mode 100644 index 893135275..000000000 --- a/docs/classic_client/snippets_table.py +++ /dev/null @@ -1,1340 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2018, Google LLC -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Testable usage examples for Google Cloud Bigtable API wrapper - -Each example function takes a ``client`` argument (which must be an instance -of :class:`google.cloud.bigtable.client.Client`) and uses it to perform a task -with the API. - -To facilitate running the examples as system tests, each example is also passed -a ``to_delete`` list; the function adds to the list any objects created which -need to be deleted during teardown. - -.. note:: - This file is under progress and will be updated with more guidance from - the team. Unit tests will be added with guidance from the team. - -""" - -import datetime -import pytest - -from google.api_core.exceptions import TooManyRequests -from google.api_core.exceptions import ServiceUnavailable -from test_utils.system import unique_resource_id -from test_utils.retry import RetryErrors - -from google.cloud._helpers import UTC -from google.cloud.bigtable import Client -from google.cloud.bigtable import enums -from google.cloud.bigtable import column_family - - -INSTANCE_ID = "snippet" + unique_resource_id("-") -CLUSTER_ID = "clus-1" + unique_resource_id("-") -TABLE_ID = "tabl-1" + unique_resource_id("-") -COLUMN_FAMILY_ID = "col_fam_id-" + unique_resource_id("-") -LOCATION_ID = "us-central1-f" -ALT_LOCATION_ID = "us-central1-a" -PRODUCTION = enums.Instance.Type.PRODUCTION -SERVER_NODES = 3 -STORAGE_TYPE = enums.StorageType.SSD -LABEL_KEY = "python-snippet" -LABEL_STAMP = ( - datetime.datetime.utcnow() - .replace(microsecond=0, tzinfo=UTC) - .strftime("%Y-%m-%dt%H-%M-%S") -) -LABELS = {LABEL_KEY: str(LABEL_STAMP)} -COLUMN_FAMILY_ID = "col_fam_id1" -COL_NAME1 = b"col-name1" -CELL_VAL1 = b"cell-val" -ROW_KEY1 = b"row_key_id1" -COLUMN_FAMILY_ID2 = "col_fam_id2" -COL_NAME2 = b"col-name2" -CELL_VAL2 = b"cell-val2" -ROW_KEY2 = b"row_key_id2" - -retry_429_503 = RetryErrors((ServiceUnavailable, TooManyRequests), max_tries=9) - - -class Config(object): - """Run-time configuration to be modified at set-up. - - This is a mutable stand-in to allow test set-up to modify - global state. - """ - - CLIENT = None - INSTANCE = None - TABLE = None - - -def setup_module(): - client = Config.CLIENT = Client(admin=True) - Config.INSTANCE = client.instance( - INSTANCE_ID, instance_type=PRODUCTION, labels=LABELS - ) - cluster = Config.INSTANCE.cluster( - CLUSTER_ID, - location_id=LOCATION_ID, - serve_nodes=SERVER_NODES, - default_storage_type=STORAGE_TYPE, - ) - operation = Config.INSTANCE.create(clusters=[cluster]) - # We want to make sure the operation completes. - operation.result(timeout=100) - Config.TABLE = Config.INSTANCE.table(TABLE_ID) - Config.TABLE.create() - gc_rule = column_family.MaxVersionsGCRule(2) - column_family1 = Config.TABLE.column_family(COLUMN_FAMILY_ID, gc_rule=gc_rule) - column_family1.create() - gc_rule2 = column_family.MaxVersionsGCRule(4) - column_family2 = Config.TABLE.column_family(COLUMN_FAMILY_ID2, gc_rule=gc_rule2) - column_family2.create() - - -def teardown_module(): - retry_429_503(Config.INSTANCE.delete)() - - -def test_bigtable_create_table(): - # [START bigtable_api_create_table] - from google.cloud.bigtable import Client - from google.cloud.bigtable import column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - - # Create table without Column families. - table1 = instance.table("table_id1") - table1.create() - - # Create table with Column families. - table2 = instance.table("table_id2") - # Define the GC policy to retain only the most recent 2 versions. - max_versions_rule = column_family.MaxVersionsGCRule(2) - table2.create(column_families={"cf1": max_versions_rule}) - - # [END bigtable_api_create_table] - assert table1.exists() - assert table2.exists() - table1.delete() - table2.delete() - - -def test_bigtable_sample_row_keys(): - table_sample = Config.INSTANCE.table("table_id1_samplerow") - initial_split_keys = [b"split_key_1", b"split_key_10", b"split_key_20"] - table_sample.create(initial_split_keys=initial_split_keys) - assert table_sample.exists() - - # [START bigtable_api_sample_row_keys] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - - table = instance.table("table_id1_samplerow") - data = table.sample_row_keys() - actual_keys, offset = zip(*[(rk.row_key, rk.offset_bytes) for rk in data]) - # [END bigtable_api_sample_row_keys] - initial_split_keys.append(b"") - assert list(actual_keys) == initial_split_keys - table.delete() - - -def test_bigtable_write_read_drop_truncate(): - # [START bigtable_api_mutate_rows] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row_keys = [ - b"row_key_1", - b"row_key_2", - b"row_key_3", - b"row_key_4", - b"row_key_20", - b"row_key_22", - b"row_key_200", - ] - col_name = b"col-name1" - rows = [] - for i, row_key in enumerate(row_keys): - value = "value_{}".format(i).encode() - row = table.row(row_key) - row.set_cell( - COLUMN_FAMILY_ID, col_name, value, timestamp=datetime.datetime.utcnow() - ) - rows.append(row) - response = table.mutate_rows(rows) - # validate that all rows written successfully - for i, status in enumerate(response): - if status.code != 0: - print("Row number {} failed to write".format(i)) - # [END bigtable_api_mutate_rows] - assert len(response) == len(rows) - # [START bigtable_api_read_row] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row_key = "row_key_1" - row = table.read_row(row_key) - # [END bigtable_api_read_row] - assert row.row_key.decode("utf-8") == row_key - # [START bigtable_api_read_rows] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - # Read full table - partial_rows = table.read_rows() - - # Read row's value - total_rows = [] - for row in partial_rows: - cell = row.cells[COLUMN_FAMILY_ID][col_name][0] - print(cell.value.decode("utf-8")) - total_rows.append(cell) - # [END bigtable_api_read_rows] - assert len(total_rows) == len(rows) - # [START bigtable_api_drop_by_prefix] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row_key_prefix = b"row_key_2" - table.drop_by_prefix(row_key_prefix, timeout=200) - # [END bigtable_api_drop_by_prefix] - dropped_row_keys = [b"row_key_2", b"row_key_20", b"row_key_22", b"row_key_200"] - for row in table.read_rows(): - assert row.row_key.decode("utf-8") not in dropped_row_keys - - # [START bigtable_api_truncate_table] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - table.truncate(timeout=200) - # [END bigtable_api_truncate_table] - rows_data_after_truncate = [] - for row in table.read_rows(): - rows_data_after_truncate.append(row.row_key) - assert rows_data_after_truncate == [] - - -def test_bigtable_mutations_batcher(): - # [START bigtable_api_mutations_batcher] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - batcher = table.mutations_batcher() - # [END bigtable_api_mutations_batcher] - - # Below code will be used while creating batcher.py snippets. - # So not removing this code as of now. - row_keys = [ - b"row_key_1", - b"row_key_2", - b"row_key_3", - b"row_key_4", - b"row_key_20", - b"row_key_22", - b"row_key_200", - ] - column_name = "column_name".encode() - # Add a single row - row_key = row_keys[0] - row = table.row(row_key) - row.set_cell( - COLUMN_FAMILY_ID, column_name, "value-0", timestamp=datetime.datetime.utcnow() - ) - batcher.mutate(row) - # Add a collections of rows - rows = [] - for i in range(1, len(row_keys)): - row = table.row(row_keys[i]) - value = "value_{}".format(i).encode() - row.set_cell( - COLUMN_FAMILY_ID, column_name, value, timestamp=datetime.datetime.utcnow() - ) - rows.append(row) - batcher.mutate_rows(rows) - # batcher will flush current batch if it - # reaches the max flush_count - - # Manually send the current batch to Cloud Bigtable - batcher.flush() - rows_on_table = [] - for row in table.read_rows(): - rows_on_table.append(row.row_key) - assert len(rows_on_table) == len(row_keys) - table.truncate(timeout=200) - - -def test_bigtable_table_column_family(): - # [START bigtable_api_table_column_family] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - - table = instance.table(TABLE_ID) - column_family_obj = table.column_family(COLUMN_FAMILY_ID) - # [END bigtable_api_table_column_family] - - assert column_family_obj.column_family_id == COLUMN_FAMILY_ID - - -def test_bigtable_list_tables(): - # [START bigtable_api_list_tables] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - tables_list = instance.list_tables() - # [END bigtable_api_list_tables] - assert len(tables_list) != 0 - - -def test_bigtable_table_name(): - import re - - # [START bigtable_api_table_name] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - - table = instance.table(TABLE_ID) - table_name = table.name - # [END bigtable_api_table_name] - _table_name_re = re.compile( - r"^projects/(?P[^/]+)/" - r"instances/(?P[^/]+)/tables/" - r"(?P[_a-zA-Z0-9][-_.a-zA-Z0-9]*)$" - ) - assert _table_name_re.match(table_name) - - -def test_bigtable_list_column_families(): - # [START bigtable_api_list_column_families] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - column_family_list = table.list_column_families() - # [END bigtable_api_list_column_families] - - assert len(column_family_list) > 0 - - -def test_bigtable_get_cluster_states(): - # [START bigtable_api_get_cluster_states] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - get_cluster_states = table.get_cluster_states() - # [END bigtable_api_get_cluster_states] - - assert CLUSTER_ID in get_cluster_states - - -def test_bigtable_table_test_iam_permissions(): - table_policy = Config.INSTANCE.table("table_id_iam_policy") - table_policy.create() - assert table_policy.exists - - # [START bigtable_api_table_test_iam_permissions] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table("table_id_iam_policy") - - permissions = ["bigtable.tables.mutateRows", "bigtable.tables.readRows"] - permissions_allowed = table.test_iam_permissions(permissions) - # [END bigtable_api_table_test_iam_permissions] - assert permissions_allowed == permissions - - -def test_bigtable_table_set_iam_policy_then_get_iam_policy(): - table_policy = Config.INSTANCE.table("table_id_iam_policy") - assert table_policy.exists - service_account_email = Config.CLIENT._credentials.service_account_email - - # [START bigtable_api_table_set_iam_policy] - from google.cloud.bigtable import Client - from google.cloud.bigtable.policy import Policy - from google.cloud.bigtable.policy import BIGTABLE_ADMIN_ROLE - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table("table_id_iam_policy") - new_policy = Policy() - new_policy[BIGTABLE_ADMIN_ROLE] = [Policy.service_account(service_account_email)] - - policy_latest = table.set_iam_policy(new_policy) - # [END bigtable_api_table_set_iam_policy] - assert len(policy_latest.bigtable_admins) > 0 - - # [START bigtable_api_table_get_iam_policy] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table("table_id_iam_policy") - policy = table.get_iam_policy() - # [END bigtable_api_table_get_iam_policy] - assert len(policy.bigtable_admins) > 0 - - -def test_bigtable_table_exists(): - # [START bigtable_api_check_table_exists] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - table_exists = table.exists() - # [END bigtable_api_check_table_exists] - assert table_exists - - -def test_bigtable_delete_table(): - table_del = Config.INSTANCE.table("table_id_del") - table_del.create() - assert table_del.exists() - - # [START bigtable_api_delete_table] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table("table_id_del") - - table.delete() - # [END bigtable_api_delete_table] - assert not table.exists() - - -def test_bigtable_table_row(): - # [START bigtable_api_table_row] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_keys = [b"row_key_1", b"row_key_2"] - row1_obj = table.row(row_keys[0]) - row2_obj = table.row(row_keys[1]) - # [END bigtable_api_table_row] - - row1_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - row1_obj.commit() - row2_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - row2_obj.commit() - - written_row_keys = [] - for row in table.read_rows(): - written_row_keys.append(row.row_key) - - assert written_row_keys == row_keys - - table.truncate(timeout=300) - - -def test_bigtable_table_append_row(): - # [START bigtable_api_table_append_row] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_keys = [b"row_key_1", b"row_key_2"] - row1_obj = table.append_row(row_keys[0]) - row2_obj = table.append_row(row_keys[1]) - # [END bigtable_api_table_append_row] - - row1_obj.append_cell_value(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - row1_obj.commit() - row2_obj.append_cell_value(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - row2_obj.commit() - - written_row_keys = [] - for row in table.read_rows(): - written_row_keys.append(row.row_key) - - assert written_row_keys == row_keys - - table.truncate(timeout=300) - - -def test_bigtable_table_direct_row(): - # [START bigtable_api_table_direct_row] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_keys = [b"row_key_1", b"row_key_2"] - row1_obj = table.direct_row(row_keys[0]) - row2_obj = table.direct_row(row_keys[1]) - # [END bigtable_api_table_direct_row] - - row1_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - row1_obj.commit() - row2_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - row2_obj.commit() - - written_row_keys = [] - for row in table.read_rows(): - written_row_keys.append(row.row_key) - - assert written_row_keys == row_keys - - table.truncate(timeout=300) - - -def test_bigtable_table_conditional_row(): - # [START bigtable_api_table_conditional_row] - from google.cloud.bigtable import Client - from google.cloud.bigtable.row_filters import PassAllFilter - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_keys = [b"row_key_1", b"row_key_2"] - filter_ = PassAllFilter(True) - row1_obj = table.conditional_row(row_keys[0], filter_=filter_) - row2_obj = table.conditional_row(row_keys[1], filter_=filter_) - # [END bigtable_api_table_conditional_row] - - row1_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1, state=False) - row1_obj.commit() - row2_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1, state=False) - row2_obj.commit() - - written_row_keys = [] - for row in table.read_rows(): - written_row_keys.append(row.row_key) - - assert written_row_keys == row_keys - - table.truncate(timeout=300) - - -def test_bigtable_column_family_name(): - # [START bigtable_api_column_family_name] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - column_families = table.list_column_families() - column_family_obj = column_families[COLUMN_FAMILY_ID] - column_family_name = column_family_obj.name - # [END bigtable_api_column_family_name] - import re - - _cf_name_re = re.compile( - r"^projects/(?P[^/]+)/" - r"instances/(?P[^/]+)/tables/" - r"(?P[^/]+)/columnFamilies/" - r"(?P[_a-zA-Z0-9][-_.a-zA-Z0-9]*)$" - ) - assert _cf_name_re.match(column_family_name) - - -def test_bigtable_create_update_delete_column_family(): - # [START bigtable_api_create_column_family] - from google.cloud.bigtable import Client - from google.cloud.bigtable import column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - column_family_id = "column_family_id1" - gc_rule = column_family.MaxVersionsGCRule(2) - column_family_obj = table.column_family(column_family_id, gc_rule=gc_rule) - column_family_obj.create() - - # [END bigtable_api_create_column_family] - column_families = table.list_column_families() - assert column_families[column_family_id].gc_rule == gc_rule - - # [START bigtable_api_update_column_family] - from google.cloud.bigtable import Client - from google.cloud.bigtable import column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - # Already existing column family id - column_family_id = "column_family_id1" - # Define the GC rule to retain data with max age of 5 days - max_age_rule = column_family.MaxAgeGCRule(datetime.timedelta(days=5)) - column_family_obj = table.column_family(column_family_id, gc_rule=max_age_rule) - column_family_obj.update() - # [END bigtable_api_update_column_family] - - updated_families = table.list_column_families() - assert updated_families[column_family_id].gc_rule == max_age_rule - - # [START bigtable_api_delete_column_family] - from google.cloud.bigtable import Client - from google.cloud.bigtable import column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - column_family_id = "column_family_id1" - column_family_obj = table.column_family(column_family_id) - column_family_obj.delete() - # [END bigtable_api_delete_column_family] - column_families = table.list_column_families() - assert column_family_id not in column_families - - -def test_bigtable_add_row_add_row_range_add_row_range_from_keys(): - row_keys = [ - b"row_key_1", - b"row_key_2", - b"row_key_3", - b"row_key_4", - b"row_key_5", - b"row_key_6", - b"row_key_7", - b"row_key_8", - b"row_key_9", - ] - - rows = [] - for row_key in row_keys: - row = Config.TABLE.row(row_key) - row.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - rows.append(row) - Config.TABLE.mutate_rows(rows) - - # [START bigtable_api_add_row_key] - from google.cloud.bigtable import Client - from google.cloud.bigtable.row_set import RowSet - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_set = RowSet() - row_set.add_row_key(b"row_key_5") - # [END bigtable_api_add_row_key] - - read_rows = table.read_rows(row_set=row_set) - expected_row_keys = [b"row_key_5"] - found_row_keys = [row.row_key for row in read_rows] - assert found_row_keys == expected_row_keys - - # [START bigtable_api_add_row_range] - from google.cloud.bigtable import Client - from google.cloud.bigtable.row_set import RowSet - from google.cloud.bigtable.row_set import RowRange - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_set = RowSet() - row_set.add_row_range(RowRange(start_key=b"row_key_3", end_key=b"row_key_7")) - # [END bigtable_api_add_row_range] - - read_rows = table.read_rows(row_set=row_set) - expected_row_keys = [b"row_key_3", b"row_key_4", b"row_key_5", b"row_key_6"] - found_row_keys = [row.row_key for row in read_rows] - assert found_row_keys == expected_row_keys - - # [START bigtable_api_row_range_from_keys] - from google.cloud.bigtable import Client - from google.cloud.bigtable.row_set import RowSet - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_set = RowSet() - row_set.add_row_range_from_keys(start_key=b"row_key_3", end_key=b"row_key_7") - # [END bigtable_api_row_range_from_keys] - - read_rows = table.read_rows(row_set=row_set) - expected_row_keys = [b"row_key_3", b"row_key_4", b"row_key_5", b"row_key_6"] - found_row_keys = [row.row_key for row in read_rows] - assert found_row_keys == expected_row_keys - table.truncate(timeout=200) - - -def test_bigtable_add_row_range_with_prefix(): - row_keys = [ - b"row_key_1", - b"row_key_2", - b"row_key_3", - b"sample_row_key_1", - b"sample_row_key_2", - ] - - rows = [] - for row_key in row_keys: - row = Config.TABLE.row(row_key) - row.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - rows.append(row) - Config.TABLE.mutate_rows(rows) - - # [START bigtable_api_add_row_range_with_prefix] - from google.cloud.bigtable import Client - from google.cloud.bigtable.row_set import RowSet - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_set = RowSet() - row_set.add_row_range_with_prefix("row") - # [END bigtable_api_add_row_range_with_prefix] - - read_rows = table.read_rows(row_set=row_set) - expected_row_keys = [ - b"row_key_1", - b"row_key_2", - b"row_key_3", - ] - found_row_keys = [row.row_key for row in read_rows] - assert found_row_keys == expected_row_keys - table.truncate(timeout=200) - - -def test_bigtable_batcher_mutate_flush_mutate_rows(): - # [START bigtable_api_batcher_mutate] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - # Batcher for max row bytes, max_row_bytes=1024 is optional. - batcher = table.mutations_batcher(max_row_bytes=1024) - - # Add a single row - row_key = b"row_key_1" - row = table.row(row_key) - row.set_cell( - COLUMN_FAMILY_ID, COL_NAME1, "value-0", timestamp=datetime.datetime.utcnow() - ) - - # In batcher, mutate will flush current batch if it - # reaches the max_row_bytes - batcher.mutate(row) - batcher.flush() - # [END bigtable_api_batcher_mutate] - - # [START bigtable_api_batcher_flush] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - # Batcher for max row bytes, max_row_bytes=1024 is optional. - batcher = table.mutations_batcher(max_row_bytes=1024) - - # Add a single row - row_key = b"row_key" - row = table.row(row_key) - row.set_cell(COLUMN_FAMILY_ID, COL_NAME1, "value-0") - - # In batcher, mutate will flush current batch if it - # reaches the max_row_bytes - batcher.mutate(row) - batcher.flush() - # [END bigtable_api_batcher_flush] - - rows_on_table = [] - for row in table.read_rows(): - rows_on_table.append(row.row_key) - assert len(rows_on_table) == 2 - table.truncate(timeout=200) - - # [START bigtable_api_batcher_mutate_rows] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - batcher = table.mutations_batcher() - - row1 = table.row(b"row_key_1") - row2 = table.row(b"row_key_2") - row3 = table.row(b"row_key_3") - row4 = table.row(b"row_key_4") - - row1.set_cell(COLUMN_FAMILY_ID, COL_NAME1, b"cell-val1") - row2.set_cell(COLUMN_FAMILY_ID, COL_NAME1, b"cell-val2") - row3.set_cell(COLUMN_FAMILY_ID, COL_NAME1, b"cell-val3") - row4.set_cell(COLUMN_FAMILY_ID, COL_NAME1, b"cell-val4") - - batcher.mutate_rows([row1, row2, row3, row4]) - - # batcher will flush current batch if it - # reaches the max flush_count - # Manually send the current batch to Cloud Bigtable - batcher.flush() - # [END bigtable_api_batcher_mutate_rows] - - rows_on_table = [] - for row in table.read_rows(): - rows_on_table.append(row.row_key) - assert len(rows_on_table) == 4 - table.truncate(timeout=200) - - -def test_bigtable_create_family_gc_max_age(): - # [START bigtable_api_create_family_gc_max_age] - from google.cloud.bigtable import Client - from google.cloud.bigtable import column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - # Define the GC rule to retain data with max age of 5 days - max_age_rule = column_family.MaxAgeGCRule(datetime.timedelta(days=5)) - - column_family_obj = table.column_family("cf1", max_age_rule) - column_family_obj.create() - - # [END bigtable_api_create_family_gc_max_age] - rule = str(column_family_obj.to_pb()) - assert "max_age" in rule - assert "seconds: 432000" in rule - column_family_obj.delete() - - -def test_bigtable_create_family_gc_max_versions(): - # [START bigtable_api_create_family_gc_max_versions] - from google.cloud.bigtable import Client - from google.cloud.bigtable import column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - # Define the GC policy to retain only the most recent 2 versions - max_versions_rule = column_family.MaxVersionsGCRule(2) - - column_family_obj = table.column_family("cf2", max_versions_rule) - column_family_obj.create() - - # [END bigtable_api_create_family_gc_max_versions] - rule = str(column_family_obj.to_pb()) - assert "max_num_versions: 2" in rule - column_family_obj.delete() - - -def test_bigtable_create_family_gc_union(): - # [START bigtable_api_create_family_gc_union] - from google.cloud.bigtable import Client - from google.cloud.bigtable import column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - max_versions_rule = column_family.MaxVersionsGCRule(2) - max_age_rule = column_family.MaxAgeGCRule(datetime.timedelta(days=5)) - - union_rule = column_family.GCRuleUnion([max_versions_rule, max_age_rule]) - - column_family_obj = table.column_family("cf3", union_rule) - column_family_obj.create() - - # [END bigtable_api_create_family_gc_union] - rule = str(column_family_obj.to_pb()) - assert "union" in rule - assert "max_age" in rule - assert "seconds: 432000" in rule - assert "max_num_versions: 2" in rule - column_family_obj.delete() - - -def test_bigtable_create_family_gc_intersection(): - # [START bigtable_api_create_family_gc_intersection] - from google.cloud.bigtable import Client - from google.cloud.bigtable import column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - max_versions_rule = column_family.MaxVersionsGCRule(2) - max_age_rule = column_family.MaxAgeGCRule(datetime.timedelta(days=5)) - - intersection_rule = column_family.GCRuleIntersection( - [max_versions_rule, max_age_rule] - ) - - column_family_obj = table.column_family("cf4", intersection_rule) - column_family_obj.create() - - # [END bigtable_api_create_family_gc_intersection] - - rule = str(column_family_obj.to_pb()) - assert "intersection" in rule - assert "max_num_versions: 2" in rule - assert "max_age" in rule - assert "seconds: 432000" in rule - column_family_obj.delete() - - -def test_bigtable_create_family_gc_nested(): - # [START bigtable_api_create_family_gc_nested] - from google.cloud.bigtable import Client - from google.cloud.bigtable import column_family - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - # Create a column family with nested GC policies. - # Create a nested GC rule: - # Drop cells that are either older than the 10 recent versions - # OR - # Drop cells that are older than a month AND older than the - # 2 recent versions - rule1 = column_family.MaxVersionsGCRule(10) - rule2 = column_family.GCRuleIntersection( - [ - column_family.MaxAgeGCRule(datetime.timedelta(days=5)), - column_family.MaxVersionsGCRule(2), - ] - ) - - nested_rule = column_family.GCRuleUnion([rule1, rule2]) - - column_family_obj = table.column_family("cf5", nested_rule) - column_family_obj.create() - - # [END bigtable_api_create_family_gc_nested] - - rule = str(column_family_obj.to_pb()) - assert "intersection" in rule - assert "max_num_versions: 2" in rule - assert "max_age" in rule - assert "seconds: 432000" in rule - column_family_obj.delete() - - -def test_bigtable_row_data_cells_cell_value_cell_values(): - value = b"value_in_col1" - row = Config.TABLE.row(b"row_key_1") - row.set_cell( - COLUMN_FAMILY_ID, COL_NAME1, value, timestamp=datetime.datetime.utcnow() - ) - row.commit() - - row.set_cell( - COLUMN_FAMILY_ID, COL_NAME1, value, timestamp=datetime.datetime.utcnow() - ) - row.commit() - - # [START bigtable_api_row_data_cells] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row_key = "row_key_1" - row_data = table.read_row(row_key) - - cells = row_data.cells - # [END bigtable_api_row_data_cells] - - actual_cell_value = cells[COLUMN_FAMILY_ID][COL_NAME1][0].value - assert actual_cell_value == value - - # [START bigtable_api_row_cell_value] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row_key = "row_key_1" - row_data = table.read_row(row_key) - - cell_value = row_data.cell_value(COLUMN_FAMILY_ID, COL_NAME1) - # [END bigtable_api_row_cell_value] - assert cell_value == value - - # [START bigtable_api_row_cell_values] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row_key = "row_key_1" - row_data = table.read_row(row_key) - - cell_values = row_data.cell_values(COLUMN_FAMILY_ID, COL_NAME1) - # [END bigtable_api_row_cell_values] - - for actual_value, timestamp in cell_values: - assert actual_value == value - - value2 = b"value_in_col2" - row.set_cell(COLUMN_FAMILY_ID, COL_NAME2, value2) - row.commit() - - # [START bigtable_api_row_find_cells] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row_key = "row_key_1" - row = table.read_row(row_key) - - cells = row.find_cells(COLUMN_FAMILY_ID, COL_NAME2) - # [END bigtable_api_row_find_cells] - - assert cells[0].value == value2 - table.truncate(timeout=200) - - -def test_bigtable_row_setcell_rowkey(): - # [START bigtable_api_row_set_cell] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row = table.row(ROW_KEY1) - - cell_val = b"cell-val" - row.set_cell( - COLUMN_FAMILY_ID, COL_NAME1, cell_val, timestamp=datetime.datetime.utcnow() - ) - # [END bigtable_api_row_set_cell] - - response = table.mutate_rows([row]) - # validate that all rows written successfully - for i, status in enumerate(response): - assert status.code == 0 - - # [START bigtable_api_row_row_key] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row = table.row(ROW_KEY1) - row_key = row.row_key - # [END bigtable_api_row_row_key] - assert row_key == ROW_KEY1 - - # [START bigtable_api_row_table] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row = table.row(ROW_KEY1) - table1 = row.table - # [END bigtable_api_row_table] - - assert table1 == table - table.truncate(timeout=200) - - -def test_bigtable_row_delete(): - table_row_del = Config.INSTANCE.table(TABLE_ID) - row_obj = table_row_del.row(b"row_key_1") - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, b"cell-val") - row_obj.commit() - written_row_keys = [] - for row in table_row_del.read_rows(): - written_row_keys.append(row.row_key) - assert written_row_keys == [b"row_key_1"] - - # [START bigtable_api_row_delete] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_key = b"row_key_1" - row_obj = table.row(row_key) - - row_obj.delete() - row_obj.commit() - # [END bigtable_api_row_delete] - - written_row_keys = [] - for row in table.read_rows(): - written_row_keys.append(row.row_key) - assert len(written_row_keys) == 0 - - -def test_bigtable_row_delete_cell(): - table_row_del_cell = Config.INSTANCE.table(TABLE_ID) - row_key1 = b"row_key_1" - row_obj = table_row_del_cell.row(row_key1) - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - row_obj.commit() - - written_row_keys = [] - for row in table_row_del_cell.read_rows(): - written_row_keys.append(row.row_key) - assert written_row_keys == [row_key1] - - # [START bigtable_api_row_delete_cell] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_key = b"row_key_1" - row_obj = table.row(row_key) - - row_obj.delete_cell(COLUMN_FAMILY_ID, COL_NAME1) - row_obj.commit() - # [END bigtable_api_row_delete_cell] - - for row in table.read_rows(): - assert not row.row_key - - -def test_bigtable_row_delete_cells(): - table_row_del_cells = Config.INSTANCE.table(TABLE_ID) - row_key1 = b"row_key_1" - row_obj = table_row_del_cells.row(row_key1) - - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, CELL_VAL1) - row_obj.commit() - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME2, CELL_VAL2) - row_obj.commit() - - written_row_keys = [] - for row in table_row_del_cells.read_rows(): - written_row_keys.append(row.row_key) - assert written_row_keys == [row_key1] - - # [START bigtable_api_row_delete_cells] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_key = b"row_key_1" - row_obj = table.row(row_key) - - row_obj.delete_cells(COLUMN_FAMILY_ID, [COL_NAME1, COL_NAME2]) - row_obj.commit() - # [END bigtable_api_row_delete_cells] - - for row in table.read_rows(): - assert not row.row_key - - -def test_bigtable_row_clear(): - table_row_clear = Config.INSTANCE.table(TABLE_ID) - row_obj = table_row_clear.row(b"row_key_1") - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, b"cell-val") - - mutation_size = row_obj.get_mutations_size() - assert mutation_size > 0 - - # [START bigtable_api_row_clear] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_key = b"row_key_1" - row_obj = table.row(row_key) - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, b"cell-val") - - row_obj.clear() - # [END bigtable_api_row_clear] - - mutation_size = row_obj.get_mutations_size() - assert mutation_size == 0 - - -def test_bigtable_row_clear_get_mutations_size(): - # [START bigtable_api_row_get_mutations_size] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_key_id = b"row_key_1" - row_obj = table.row(row_key_id) - - mutation_size = row_obj.get_mutations_size() - # [END bigtable_api_row_get_mutations_size] - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, b"cell-val") - mutation_size = row_obj.get_mutations_size() - assert mutation_size > 0 - - row_obj.clear() - mutation_size = row_obj.get_mutations_size() - assert mutation_size == 0 - - -def test_bigtable_row_setcell_commit_rowkey(): - # [START bigtable_api_row_set_cell] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_key = b"row_key_1" - cell_val = b"cell-val" - row_obj = table.row(row_key) - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, cell_val) - # [END bigtable_api_row_set_cell] - row_obj.commit() - - # [START bigtable_api_row_commit] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_key = b"row_key_2" - cell_val = b"cell-val" - row_obj = table.row(row_key) - row_obj.set_cell(COLUMN_FAMILY_ID, COL_NAME1, cell_val) - row_obj.commit() - # [END bigtable_api_row_commit] - - written_row_keys = [] - for row in table.read_rows(): - written_row_keys.append(row.row_key) - - assert written_row_keys == [b"row_key_1", b"row_key_2"] - - # [START bigtable_api_row_row_key] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - - row_key_id = b"row_key_2" - row_obj = table.row(row_key_id) - row_key = row_obj.row_key - # [END bigtable_api_row_row_key] - assert row_key == row_key_id - table.truncate(timeout=300) - - -def test_bigtable_row_append_cell_value(): - row = Config.TABLE.row(ROW_KEY1) - - cell_val1 = b"1" - row.set_cell(COLUMN_FAMILY_ID, COL_NAME1, cell_val1) - row.commit() - - # [START bigtable_api_row_append_cell_value] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row = table.row(ROW_KEY1, append=True) - - cell_val2 = b"2" - row.append_cell_value(COLUMN_FAMILY_ID, COL_NAME1, cell_val2) - # [END bigtable_api_row_append_cell_value] - row.commit() - - row_data = table.read_row(ROW_KEY1) - actual_value = row_data.cell_value(COLUMN_FAMILY_ID, COL_NAME1) - assert actual_value == cell_val1 + cell_val2 - - # [START bigtable_api_row_commit] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row = Config.TABLE.row(ROW_KEY2) - cell_val = 1 - row.set_cell(COLUMN_FAMILY_ID, COL_NAME1, cell_val) - row.commit() - # [END bigtable_api_row_commit] - - # [START bigtable_api_row_increment_cell_value] - from google.cloud.bigtable import Client - - client = Client(admin=True) - instance = client.instance(INSTANCE_ID) - table = instance.table(TABLE_ID) - row = table.row(ROW_KEY2, append=True) - - int_val = 3 - row.increment_cell_value(COLUMN_FAMILY_ID, COL_NAME1, int_val) - # [END bigtable_api_row_increment_cell_value] - row.commit() - - row_data = table.read_row(ROW_KEY2) - actual_value = row_data.cell_value(COLUMN_FAMILY_ID, COL_NAME1) - - import struct - - _PACK_I64 = struct.Struct(">q").pack - assert actual_value == _PACK_I64(cell_val + int_val) - table.truncate(timeout=200) - - -if __name__ == "__main__": - pytest.main() diff --git a/docs/classic_client/table-api.rst b/docs/classic_client/table-api.rst deleted file mode 100644 index 1bbf85146..000000000 --- a/docs/classic_client/table-api.rst +++ /dev/null @@ -1,153 +0,0 @@ -Table Admin API -=============== - -After creating an :class:`Instance `, you can -interact with individual tables, groups of tables or column families within -a table. - -List Tables ------------ - -If you want a comprehensive list of all existing tables in a instance, make a -`ListTables`_ API request with -:meth:`Instance.list_tables() `: - -.. code:: python - - >>> instance.list_tables() - [, - ] - -Table Factory -------------- - -To create a :class:`Table ` object: - -.. code:: python - - table = instance.table(table_id) - -Even if this :class:`Table ` already -has been created with the API, you'll want this object to use as a -parent of a :class:`ColumnFamily ` -or :class:`Row `. - -Create a new Table ------------------- - -After creating the table object, make a `CreateTable`_ API request -with :meth:`create() `: - -.. code:: python - - table.create() - -If you would like to initially split the table into several tablets (tablets are -similar to HBase regions): - -.. code:: python - - table.create(initial_split_keys=['s1', 's2']) - -Delete an existing Table ------------------------- - -Make a `DeleteTable`_ API request with -:meth:`delete() `: - -.. code:: python - - table.delete() - -List Column Families in a Table -------------------------------- - -Though there is no **official** method for retrieving `column families`_ -associated with a table, the `GetTable`_ API method returns a -table object with the names of the column families. - -To retrieve the list of column families use -:meth:`list_column_families() `: - -.. code:: python - - column_families = table.list_column_families() - -Column Family Factory ---------------------- - -To create a -:class:`ColumnFamily ` object: - -.. code:: python - - column_family = table.column_family(column_family_id) - -There is no real reason to use this factory unless you intend to -create or delete a column family. - -In addition, you can specify an optional ``gc_rule`` (a -:class:`GarbageCollectionRule ` -or similar): - -.. code:: python - - column_family = table.column_family(column_family_id, - gc_rule=gc_rule) - -This rule helps the backend determine when and how to clean up old cells -in the column family. - -See :doc:`column-family` for more information about -:class:`GarbageCollectionRule ` -and related classes. - -Create a new Column Family --------------------------- - -After creating the column family object, make a `CreateColumnFamily`_ API -request with -:meth:`ColumnFamily.create() ` - -.. code:: python - - column_family.create() - -Delete an existing Column Family --------------------------------- - -Make a `DeleteColumnFamily`_ API request with -:meth:`ColumnFamily.delete() ` - -.. code:: python - - column_family.delete() - -Update an existing Column Family --------------------------------- - -Make an `UpdateColumnFamily`_ API request with -:meth:`ColumnFamily.delete() ` - -.. code:: python - - column_family.update() - -Next Step ---------- - -Now we go down the final step of the hierarchy from -:class:`Table ` to -:class:`Row ` as well as streaming -data directly via a :class:`Table `. - -Head next to learn about the :doc:`data-api`. - -.. _ListTables: https://googleapis.dev/python/bigtable/latest/table-api.html#list-tables -.. _CreateTable: https://googleapis.dev/python/bigtable/latest/table-api.html#create-a-new-table -.. _DeleteTable: https://googleapis.dev/python/bigtable/latest/table-api.html#delete-an-existing-table -.. _GetTable: https://github.com/googleapis/python-bigtable/blob/main/google/cloud/bigtable_admin_v2/proto/bigtable_table_admin.proto#L97-L102 -.. _CreateColumnFamily: https://googleapis.dev/python/bigtable/latest/table-api.html?highlight=gettable#create-a-new-column-family -.. _UpdateColumnFamily: https://googleapis.dev/python/bigtable/latest/table-api.html?highlight=gettable#update-an-existing-column-family -.. _DeleteColumnFamily: https://googleapis.dev/python/bigtable/latest/table-api.html?highlight=gettable#delete-an-existing-column-family -.. _column families: https://cloud.google.com/bigtable/docs/schema-design#column_families_and_column_qualifiers diff --git a/docs/classic_client/table.rst b/docs/classic_client/table.rst deleted file mode 100644 index c230725d1..000000000 --- a/docs/classic_client/table.rst +++ /dev/null @@ -1,6 +0,0 @@ -Table -~~~~~ - -.. automodule:: google.cloud.bigtable.table - :members: - :show-inheritance: diff --git a/docs/classic_client/usage.rst b/docs/classic_client/usage.rst deleted file mode 100644 index 7a47f4d4a..000000000 --- a/docs/classic_client/usage.rst +++ /dev/null @@ -1,38 +0,0 @@ -Classic Client -============== - -.. toctree:: - :maxdepth: 2 - - client-intro - - instance-api - table-api - data-api - - client - cluster - instance - table - app-profile - backup - column-family - encryption-info - row - row-data - row-filters - row-set - batcher - - -In the hierarchy of API concepts - -* a :class:`Client ` owns an - :class:`Instance ` -* an :class:`Instance ` owns a - :class:`Table ` -* a :class:`Table ` owns a - :class:`ColumnFamily ` -* a :class:`Table ` owns a - :class:`Row ` - (and all the cells in the row) diff --git a/docs/index.rst b/docs/index.rst index c7f9721f3..62f32e0fb 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,7 +8,6 @@ Client Types :maxdepth: 3 data_client/data_client_usage - classic_client/usage Changelog From c0256a711aca5d2b26d21f5ab9960694540fba42 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 7 Apr 2025 14:40:44 -0700 Subject: [PATCH 3/6] replaced docs files with new one for data client --- docs/data_client.rst | 83 +++++++++++++++++++ docs/data_client/async_data_client.rst | 12 --- .../async_data_execute_query_iterator.rst | 6 -- .../async_data_mutations_batcher.rst | 6 -- docs/data_client/async_data_table.rst | 11 --- docs/data_client/common_data_exceptions.rst | 6 -- .../common_data_execute_query_metadata.rst | 6 -- .../common_data_execute_query_values.rst | 6 -- docs/data_client/common_data_mutations.rst | 6 -- .../common_data_read_modify_write_rules.rst | 6 -- .../common_data_read_rows_query.rst | 6 -- docs/data_client/common_data_row.rst | 6 -- docs/data_client/common_data_row_filters.rst | 62 -------------- docs/data_client/data_client_usage.rst | 39 --------- docs/data_client/sync_data_client.rst | 6 -- .../sync_data_execute_query_iterator.rst | 6 -- .../sync_data_mutations_batcher.rst | 6 -- docs/data_client/sync_data_table.rst | 6 -- docs/index.rst | 2 +- 19 files changed, 84 insertions(+), 203 deletions(-) create mode 100644 docs/data_client.rst delete mode 100644 docs/data_client/async_data_client.rst delete mode 100644 docs/data_client/async_data_execute_query_iterator.rst delete mode 100644 docs/data_client/async_data_mutations_batcher.rst delete mode 100644 docs/data_client/async_data_table.rst delete mode 100644 docs/data_client/common_data_exceptions.rst delete mode 100644 docs/data_client/common_data_execute_query_metadata.rst delete mode 100644 docs/data_client/common_data_execute_query_values.rst delete mode 100644 docs/data_client/common_data_mutations.rst delete mode 100644 docs/data_client/common_data_read_modify_write_rules.rst delete mode 100644 docs/data_client/common_data_read_rows_query.rst delete mode 100644 docs/data_client/common_data_row.rst delete mode 100644 docs/data_client/common_data_row_filters.rst delete mode 100644 docs/data_client/data_client_usage.rst delete mode 100644 docs/data_client/sync_data_client.rst delete mode 100644 docs/data_client/sync_data_execute_query_iterator.rst delete mode 100644 docs/data_client/sync_data_mutations_batcher.rst delete mode 100644 docs/data_client/sync_data_table.rst diff --git a/docs/data_client.rst b/docs/data_client.rst new file mode 100644 index 000000000..df9673298 --- /dev/null +++ b/docs/data_client.rst @@ -0,0 +1,83 @@ +Bigtable Data Client Async +~~~~~~~~~~~~~~~~~~~~~~~~~~ + + .. note:: + + It is generally not recommended to use the async client in an otherwise synchronous codebase. To make use of asyncio's + performance benefits, the codebase should be designed to be async from the ground up. + + +.. automodule:: google.cloud.bigtable.data + :members: BigtableDataClientAsync + :show-inheritance: + :ignore-module-all: + +Async Classes +~~~~~~~~~~~~~ + +.. autoclass:: google.cloud.bigtable.data.BigtableDataClientAsync + :members: + :show-inheritance: + +.. autoclass:: google.cloud.bigtable.data.TableAsync + :members: + :show-inheritance: + +.. autoclass:: google.cloud.bigtable.data.MutationsBatcherAsync + :members: + :show-inheritance: + +.. autoclass:: google.cloud.bigtable.data.execute_query.ExecuteQueryIteratorAsync + :members: + :show-inheritance: + +Sync Classes +~~~~~~~~~~~~~ + +.. autoclass:: google.cloud.bigtable.data.BigtableDataClient + :members: + :show-inheritance: + +.. autoclass:: google.cloud.bigtable.data.Table + :members: + :show-inheritance: + +.. autoclass:: google.cloud.bigtable.data.MutationsBatcher + :members: + :show-inheritance: + +.. autoclass:: google.cloud.bigtable.data.execute_query.ExecuteQueryIterator + :members: + :show-inheritance: + + +Common Modules +~~~~~~~~~~~~~~ + +.. automodule:: google.cloud.bigtable.data.exceptions + :members: + :show-inheritance: + +.. automodule:: google.cloud.bigtable.data.mutations + :members: + :show-inheritance: + +.. automodule:: google.cloud.bigtable.data.read_modify_write_rules + :members: + :show-inheritance: + +.. automodule:: google.cloud.bigtable.data.read_rows_query + :members: + :show-inheritance: + +.. automodule:: google.cloud.bigtable.data.row + :members: + :show-inheritance: + +.. automodule:: google.cloud.bigtable.data.row_filters + :members: + :show-inheritance: + +.. automodule:: google.cloud.bigtable.data.execute_query + :members: + :show-inheritance: diff --git a/docs/data_client/async_data_client.rst b/docs/data_client/async_data_client.rst deleted file mode 100644 index 2ddcc090c..000000000 --- a/docs/data_client/async_data_client.rst +++ /dev/null @@ -1,12 +0,0 @@ -Bigtable Data Client Async -~~~~~~~~~~~~~~~~~~~~~~~~~~ - - .. note:: - - It is generally not recommended to use the async client in an otherwise synchronous codebase. To make use of asyncio's - performance benefits, the codebase should be designed to be async from the ground up. - - -.. autoclass:: google.cloud.bigtable.data.BigtableDataClientAsync - :members: - :show-inheritance: diff --git a/docs/data_client/async_data_execute_query_iterator.rst b/docs/data_client/async_data_execute_query_iterator.rst deleted file mode 100644 index b911fab7f..000000000 --- a/docs/data_client/async_data_execute_query_iterator.rst +++ /dev/null @@ -1,6 +0,0 @@ -Execute Query Iterator Async -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: google.cloud.bigtable.data.execute_query.ExecuteQueryIteratorAsync - :members: - :show-inheritance: diff --git a/docs/data_client/async_data_mutations_batcher.rst b/docs/data_client/async_data_mutations_batcher.rst deleted file mode 100644 index 3e81f885a..000000000 --- a/docs/data_client/async_data_mutations_batcher.rst +++ /dev/null @@ -1,6 +0,0 @@ -Mutations Batcher Async -~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: google.cloud.bigtable.data._async.mutations_batcher - :members: - :show-inheritance: diff --git a/docs/data_client/async_data_table.rst b/docs/data_client/async_data_table.rst deleted file mode 100644 index 3b7973e8e..000000000 --- a/docs/data_client/async_data_table.rst +++ /dev/null @@ -1,11 +0,0 @@ -Table Async -~~~~~~~~~~~ - - .. note:: - - It is generally not recommended to use the async client in an otherwise synchronous codebase. To make use of asyncio's - performance benefits, the codebase should be designed to be async from the ground up. - -.. autoclass:: google.cloud.bigtable.data._async.client.TableAsync - :members: - :show-inheritance: diff --git a/docs/data_client/common_data_exceptions.rst b/docs/data_client/common_data_exceptions.rst deleted file mode 100644 index 6180ef222..000000000 --- a/docs/data_client/common_data_exceptions.rst +++ /dev/null @@ -1,6 +0,0 @@ -Custom Exceptions -~~~~~~~~~~~~~~~~~ - -.. automodule:: google.cloud.bigtable.data.exceptions - :members: - :show-inheritance: diff --git a/docs/data_client/common_data_execute_query_metadata.rst b/docs/data_client/common_data_execute_query_metadata.rst deleted file mode 100644 index 69add630d..000000000 --- a/docs/data_client/common_data_execute_query_metadata.rst +++ /dev/null @@ -1,6 +0,0 @@ -Execute Query Metadata -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: google.cloud.bigtable.data.execute_query.metadata - :members: - :show-inheritance: diff --git a/docs/data_client/common_data_execute_query_values.rst b/docs/data_client/common_data_execute_query_values.rst deleted file mode 100644 index 6c4fb71c1..000000000 --- a/docs/data_client/common_data_execute_query_values.rst +++ /dev/null @@ -1,6 +0,0 @@ -Execute Query Values -~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: google.cloud.bigtable.data.execute_query.values - :members: - :show-inheritance: diff --git a/docs/data_client/common_data_mutations.rst b/docs/data_client/common_data_mutations.rst deleted file mode 100644 index 9d7a9eab2..000000000 --- a/docs/data_client/common_data_mutations.rst +++ /dev/null @@ -1,6 +0,0 @@ -Mutations -~~~~~~~~~ - -.. automodule:: google.cloud.bigtable.data.mutations - :members: - :show-inheritance: diff --git a/docs/data_client/common_data_read_modify_write_rules.rst b/docs/data_client/common_data_read_modify_write_rules.rst deleted file mode 100644 index 2f28ddf3f..000000000 --- a/docs/data_client/common_data_read_modify_write_rules.rst +++ /dev/null @@ -1,6 +0,0 @@ -Read Modify Write Rules -~~~~~~~~~~~~~~~~~~~~~~~ - -.. automodule:: google.cloud.bigtable.data.read_modify_write_rules - :members: - :show-inheritance: diff --git a/docs/data_client/common_data_read_rows_query.rst b/docs/data_client/common_data_read_rows_query.rst deleted file mode 100644 index 4e3e796d9..000000000 --- a/docs/data_client/common_data_read_rows_query.rst +++ /dev/null @@ -1,6 +0,0 @@ -Read Rows Query -~~~~~~~~~~~~~~~ - -.. automodule:: google.cloud.bigtable.data.read_rows_query - :members: - :show-inheritance: diff --git a/docs/data_client/common_data_row.rst b/docs/data_client/common_data_row.rst deleted file mode 100644 index 63bc71143..000000000 --- a/docs/data_client/common_data_row.rst +++ /dev/null @@ -1,6 +0,0 @@ -Rows and Cells -~~~~~~~~~~~~~~ - -.. automodule:: google.cloud.bigtable.data.row - :members: - :show-inheritance: diff --git a/docs/data_client/common_data_row_filters.rst b/docs/data_client/common_data_row_filters.rst deleted file mode 100644 index 22bda8a26..000000000 --- a/docs/data_client/common_data_row_filters.rst +++ /dev/null @@ -1,62 +0,0 @@ -Bigtable Row Filters -==================== - -It is possible to use a -:class:`RowFilter ` -when constructing a :class:`ReadRowsQuery ` - -The following basic filters -are provided: - -* :class:`SinkFilter <.data.row_filters.SinkFilter>` -* :class:`PassAllFilter <.data.row_filters.PassAllFilter>` -* :class:`BlockAllFilter <.data.row_filters.BlockAllFilter>` -* :class:`RowKeyRegexFilter <.data.row_filters.RowKeyRegexFilter>` -* :class:`RowSampleFilter <.data.row_filters.RowSampleFilter>` -* :class:`FamilyNameRegexFilter <.data.row_filters.FamilyNameRegexFilter>` -* :class:`ColumnQualifierRegexFilter <.data.row_filters.ColumnQualifierRegexFilter>` -* :class:`TimestampRangeFilter <.data.row_filters.TimestampRangeFilter>` -* :class:`ColumnRangeFilter <.data.row_filters.ColumnRangeFilter>` -* :class:`ValueRegexFilter <.data.row_filters.ValueRegexFilter>` -* :class:`ValueRangeFilter <.data.row_filters.ValueRangeFilter>` -* :class:`CellsRowOffsetFilter <.data.row_filters.CellsRowOffsetFilter>` -* :class:`CellsRowLimitFilter <.data.row_filters.CellsRowLimitFilter>` -* :class:`CellsColumnLimitFilter <.data.row_filters.CellsColumnLimitFilter>` -* :class:`StripValueTransformerFilter <.data.row_filters.StripValueTransformerFilter>` -* :class:`ApplyLabelFilter <.data.row_filters.ApplyLabelFilter>` - -In addition, these filters can be combined into composite filters with - -* :class:`RowFilterChain <.data.row_filters.RowFilterChain>` -* :class:`RowFilterUnion <.data.row_filters.RowFilterUnion>` -* :class:`ConditionalRowFilter <.data.row_filters.ConditionalRowFilter>` - -These rules can be nested arbitrarily, with a basic filter at the lowest -level. For example: - -.. code:: python - - # Filter in a specified column (matching any column family). - col1_filter = ColumnQualifierRegexFilter(b'columnbia') - - # Create a filter to label results. - label1 = u'label-red' - label1_filter = ApplyLabelFilter(label1) - - # Combine the filters to label all the cells in columnbia. - chain1 = RowFilterChain(filters=[col1_filter, label1_filter]) - - # Create a similar filter to label cells blue. - col2_filter = ColumnQualifierRegexFilter(b'columnseeya') - label2 = u'label-blue' - label2_filter = ApplyLabelFilter(label2) - chain2 = RowFilterChain(filters=[col2_filter, label2_filter]) - - # Bring our two labeled columns together. - row_filter = RowFilterUnion(filters=[chain1, chain2]) - ----- - -.. automodule:: google.cloud.bigtable.data.row_filters - :members: - :show-inheritance: diff --git a/docs/data_client/data_client_usage.rst b/docs/data_client/data_client_usage.rst deleted file mode 100644 index f5bbac278..000000000 --- a/docs/data_client/data_client_usage.rst +++ /dev/null @@ -1,39 +0,0 @@ -Data Client -=========== - -Sync Surface ------------- - -.. toctree:: - :maxdepth: 3 - - sync_data_client - sync_data_table - sync_data_mutations_batcher - sync_data_execute_query_iterator - -Async Surface -------------- - -.. toctree:: - :maxdepth: 3 - - async_data_client - async_data_table - async_data_mutations_batcher - async_data_execute_query_iterator - -Common Classes --------------- - -.. toctree:: - :maxdepth: 3 - - common_data_read_rows_query - common_data_row - common_data_row_filters - common_data_mutations - common_data_read_modify_write_rules - common_data_exceptions - common_data_execute_query_values - common_data_execute_query_metadata diff --git a/docs/data_client/sync_data_client.rst b/docs/data_client/sync_data_client.rst deleted file mode 100644 index cf7c00dad..000000000 --- a/docs/data_client/sync_data_client.rst +++ /dev/null @@ -1,6 +0,0 @@ -Bigtable Data Client -~~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: google.cloud.bigtable.data.BigtableDataClient - :members: - :show-inheritance: diff --git a/docs/data_client/sync_data_execute_query_iterator.rst b/docs/data_client/sync_data_execute_query_iterator.rst deleted file mode 100644 index 6eb9f84db..000000000 --- a/docs/data_client/sync_data_execute_query_iterator.rst +++ /dev/null @@ -1,6 +0,0 @@ -Execute Query Iterator -~~~~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: google.cloud.bigtable.data.execute_query.ExecuteQueryIterator - :members: - :show-inheritance: diff --git a/docs/data_client/sync_data_mutations_batcher.rst b/docs/data_client/sync_data_mutations_batcher.rst deleted file mode 100644 index 2b7d1bfe0..000000000 --- a/docs/data_client/sync_data_mutations_batcher.rst +++ /dev/null @@ -1,6 +0,0 @@ -Mutations Batcher -~~~~~~~~~~~~~~~~~ - -.. automodule:: google.cloud.bigtable.data._sync_autogen.mutations_batcher - :members: - :show-inheritance: diff --git a/docs/data_client/sync_data_table.rst b/docs/data_client/sync_data_table.rst deleted file mode 100644 index 95c91eb27..000000000 --- a/docs/data_client/sync_data_table.rst +++ /dev/null @@ -1,6 +0,0 @@ -Table -~~~~~ - -.. autoclass:: google.cloud.bigtable.data.Table - :members: - :show-inheritance: diff --git a/docs/index.rst b/docs/index.rst index 62f32e0fb..0b236d434 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,7 +7,7 @@ Client Types .. toctree:: :maxdepth: 3 - data_client/data_client_usage + data_client Changelog From 98b7db600eee6216c35516e566277a9c21db5890 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 7 Apr 2025 15:48:20 -0700 Subject: [PATCH 4/6] moved RowKeySamples and ShardedQuery into read_rows_query --- google/cloud/bigtable/data/__init__.py | 4 ++-- google/cloud/bigtable/data/_async/client.py | 4 ++-- google/cloud/bigtable/data/_helpers.py | 6 ------ google/cloud/bigtable/data/_sync_autogen/client.py | 4 ++-- google/cloud/bigtable/data/read_rows_query.py | 9 ++++++--- 5 files changed, 12 insertions(+), 15 deletions(-) diff --git a/google/cloud/bigtable/data/__init__.py b/google/cloud/bigtable/data/__init__.py index 15f9bc167..a2bfd3f3a 100644 --- a/google/cloud/bigtable/data/__init__.py +++ b/google/cloud/bigtable/data/__init__.py @@ -24,6 +24,8 @@ from google.cloud.bigtable.data.read_rows_query import ReadRowsQuery from google.cloud.bigtable.data.read_rows_query import RowRange +from google.cloud.bigtable.data.read_rows_query import RowKeySamples +from google.cloud.bigtable.data.read_rows_query import ShardedQuery from google.cloud.bigtable.data.row import Row from google.cloud.bigtable.data.row import Cell @@ -44,8 +46,6 @@ from google.cloud.bigtable.data.exceptions import ParameterTypeInferenceFailed from google.cloud.bigtable.data._helpers import TABLE_DEFAULT -from google.cloud.bigtable.data._helpers import RowKeySamples -from google.cloud.bigtable.data._helpers import ShardedQuery # setup custom CrossSync mappings for library from google.cloud.bigtable_v2.services.bigtable.async_client import ( diff --git a/google/cloud/bigtable/data/_async/client.py b/google/cloud/bigtable/data/_async/client.py index ecf481889..6c9770957 100644 --- a/google/cloud/bigtable/data/_async/client.py +++ b/google/cloud/bigtable/data/_async/client.py @@ -92,8 +92,8 @@ if TYPE_CHECKING: - from google.cloud.bigtable.data._helpers import RowKeySamples - from google.cloud.bigtable.data._helpers import ShardedQuery + from google.cloud.bigtable.data import RowKeySamples + from google.cloud.bigtable.data import ShardedQuery if CrossSync.is_async: from google.cloud.bigtable.data._async.mutations_batcher import ( diff --git a/google/cloud/bigtable/data/_helpers.py b/google/cloud/bigtable/data/_helpers.py index 4c45e5c1c..9fdf23e68 100644 --- a/google/cloud/bigtable/data/_helpers.py +++ b/google/cloud/bigtable/data/_helpers.py @@ -35,12 +35,6 @@ Helper functions used in various places in the library. """ -# Type alias for the output of sample_keys -RowKeySamples = List[Tuple[bytes, int]] - -# type alias for the output of query.shard() -ShardedQuery = List[ReadRowsQuery] - # used by read_rows_sharded to limit how many requests are attempted in parallel _CONCURRENCY_LIMIT = 10 diff --git a/google/cloud/bigtable/data/_sync_autogen/client.py b/google/cloud/bigtable/data/_sync_autogen/client.py index 492e86224..1c1511ff7 100644 --- a/google/cloud/bigtable/data/_sync_autogen/client.py +++ b/google/cloud/bigtable/data/_sync_autogen/client.py @@ -72,8 +72,8 @@ from google.cloud.bigtable.data._sync_autogen.mutations_batcher import _MB_SIZE if TYPE_CHECKING: - from google.cloud.bigtable.data._helpers import RowKeySamples - from google.cloud.bigtable.data._helpers import ShardedQuery + from google.cloud.bigtable.data import RowKeySamples + from google.cloud.bigtable.data import ShardedQuery from google.cloud.bigtable.data._sync_autogen.mutations_batcher import ( MutationsBatcher, ) diff --git a/google/cloud/bigtable/data/read_rows_query.py b/google/cloud/bigtable/data/read_rows_query.py index e0839a2af..2ea538a82 100644 --- a/google/cloud/bigtable/data/read_rows_query.py +++ b/google/cloud/bigtable/data/read_rows_query.py @@ -23,9 +23,12 @@ from google.cloud.bigtable_v2.types import RowSet as RowSetPB from google.cloud.bigtable_v2.types import ReadRowsRequest as ReadRowsRequestPB -if TYPE_CHECKING: - from google.cloud.bigtable.data import RowKeySamples - from google.cloud.bigtable.data import ShardedQuery + +RowKeySamples = list[tuple[bytes, int]] +"""List of table dividion points returned by `table.sample_row_keys()`. Used as an input for `query.shard()`.""" + +ShardedQuery = list["ReadRowsQuery"] +"""List of sharded queries returned by `query.shard()`. Used as an input for `table.read_rows_sharded()`""" class RowRange: From 4cae05c6da8850a9d7a9f4db4e97b4700bf64a0e Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 7 Apr 2025 15:55:56 -0700 Subject: [PATCH 5/6] changed heading --- docs/data_client.rst | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/data_client.rst b/docs/data_client.rst index df9673298..baf50ac53 100644 --- a/docs/data_client.rst +++ b/docs/data_client.rst @@ -1,11 +1,5 @@ -Bigtable Data Client Async -~~~~~~~~~~~~~~~~~~~~~~~~~~ - - .. note:: - - It is generally not recommended to use the async client in an otherwise synchronous codebase. To make use of asyncio's - performance benefits, the codebase should be designed to be async from the ground up. - +Data Client +~~~~~~~~~~~ .. automodule:: google.cloud.bigtable.data :members: BigtableDataClientAsync From 86e32902a39731cfa2718d4392eac376864749fb Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 7 Apr 2025 17:04:20 -0700 Subject: [PATCH 6/6] added types to Args in read_rows_stream --- google/cloud/bigtable/data/_async/client.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/google/cloud/bigtable/data/_async/client.py b/google/cloud/bigtable/data/_async/client.py index 6c9770957..73384d5d4 100644 --- a/google/cloud/bigtable/data/_async/client.py +++ b/google/cloud/bigtable/data/_async/client.py @@ -772,16 +772,16 @@ async def read_rows_stream( retryable_errors list until operation_timeout is reached. Args: - query: contains details about which rows to return - operation_timeout: the time budget for the entire operation, in seconds. + query (google.cloud.bigtable.data.read_rows_query.ReadRowsQuery): contains details about which rows to return + operation_timeout (Union[float, TABLE_DEFAULT]): the time budget for the entire operation, in seconds. Failed requests will be retried within the budget. Defaults to the Table's default_read_rows_operation_timeout - attempt_timeout: the time budget for an individual network request, in seconds. + attempt_timeout (Union[float, TABLE_DEFAULT, None]): the time budget for an individual network request, in seconds. If it takes longer than this time to complete, the request will be cancelled with a DeadlineExceeded exception, and a retry will be attempted. Defaults to the Table's default_read_rows_attempt_timeout. If None, defaults to operation_timeout. - retryable_errors: a list of errors that will be retried if encountered. + retryable_errors (Union[Sequence[type[Exception]], TABLE_DEFAULT]): a list of errors that will be retried if encountered. Defaults to the Table's default_read_rows_retryable_errors Returns: AsyncIterable[Row]: an asynchronous iterator that yields rows returned by the query @@ -940,7 +940,8 @@ async def read_rows_sharded( Returns: list[Row]: a list of Rows returned by the query Raises: - ShardedReadRowsExceptionGroup: if any of the queries failed + google.cloud.bigtable.data.exceptions.ShardedReadRowsExceptionGroup: if any + of the queries failed ValueError: if the query_list is empty """ if not sharded_query: @@ -1290,8 +1291,8 @@ async def bulk_mutate_rows( retryable_errors: a list of errors that will be retried if encountered. Defaults to the Table's default_mutate_rows_retryable_errors Raises: - MutationsExceptionGroup: if one or more mutations fails - Contains details about any failed entries in .exceptions + google.cloud.bigtable.data.exceptions.MutationsExceptionGroup: if one + or more mutations fails. Contains details about any failed entries in .exceptions ValueError: if invalid arguments are provided """ operation_timeout, attempt_timeout = _get_timeouts(