diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 10cf433a8..3f7634f25 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:8ff1efe878e18bd82a0fb7b70bb86f77e7ab6901fed394440b6135db0ba8d84a -# created: 2025-01-09T12:01:16.422459506Z + digest: sha256:f016446d6e520e5fb552c45b110cba3f217bffdd3d06bdddd076e9e6d13266cf +# created: 2025-02-21T19:32:52.01306189Z diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b37686f76..6df17303f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,8 +5,8 @@ # https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax # Note: This file is autogenerated. To make changes to the codeowner team, please update .repo-metadata.json. -# @googleapis/yoshi-python @googleapis/cloud-storage-dpe are the default owners for changes in this repo -* @googleapis/yoshi-python @googleapis/cloud-storage-dpe +# @googleapis/yoshi-python @googleapis/gcs-sdk-team are the default owners for changes in this repo +* @googleapis/yoshi-python @googleapis/gcs-sdk-team -# @googleapis/python-samples-reviewers @googleapis/cloud-storage-dpe are the default owners for samples changes -/samples/ @googleapis/python-samples-reviewers @googleapis/cloud-storage-dpe +# @googleapis/python-samples-reviewers @googleapis/gcs-sdk-team are the default owners for samples changes +/samples/ @googleapis/python-samples-reviewers @googleapis/gcs-sdk-team diff --git a/.kokoro/build.sh b/.kokoro/build.sh index fdc6d0271..2e7461228 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -15,11 +15,13 @@ set -eo pipefail +CURRENT_DIR=$(dirname "${BASH_SOURCE[0]}") + if [[ -z "${PROJECT_ROOT:-}" ]]; then - PROJECT_ROOT="github/python-storage" + PROJECT_ROOT=$(realpath "${CURRENT_DIR}/..") fi -cd "${PROJECT_ROOT}" +pushd "${PROJECT_ROOT}" # Disable buffering, so that the logs stream through. export PYTHONUNBUFFERED=1 @@ -46,10 +48,16 @@ export TEST_UNIVERSE_LOCATION=$(gcloud secrets versions access latest --project env | grep KOKORO # Setup service account credentials. -export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json +if [[ -f "${KOKORO_GFILE_DIR}/service-account.json" ]] +then + export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/service-account.json +fi # Setup project id. -export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") +if [[ -f "${KOKORO_GFILE_DIR}/project-id.json" ]] +then + export PROJECT_ID=$(cat "${KOKORO_GFILE_DIR}/project-id.json") +fi # If this is a continuous build, send the test log to the FlakyBot. # See https://github.com/googleapis/repo-automation-bots/tree/main/packages/flakybot. @@ -64,7 +72,7 @@ fi # If NOX_SESSION is set, it only runs the specified session, # otherwise run all the sessions. if [[ -n "${NOX_SESSION:-}" ]]; then - python3 -m nox -s ${NOX_SESSION:-} + python3 -m nox -s ${NOX_SESSION:-} else - python3 -m nox + python3 -m nox fi diff --git a/.kokoro/docker/docs/requirements.in b/.kokoro/docker/docs/requirements.in index 816817c67..586bd0703 100644 --- a/.kokoro/docker/docs/requirements.in +++ b/.kokoro/docker/docs/requirements.in @@ -1 +1,2 @@ nox +gcp-docuploader diff --git a/.kokoro/docker/docs/requirements.txt b/.kokoro/docker/docs/requirements.txt index f99a5c4aa..a9360a25b 100644 --- a/.kokoro/docker/docs/requirements.txt +++ b/.kokoro/docker/docs/requirements.txt @@ -2,16 +2,124 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --allow-unsafe --generate-hashes synthtool/gcp/templates/python_library/.kokoro/docker/docs/requirements.in +# pip-compile --allow-unsafe --generate-hashes requirements.in # -argcomplete==3.5.2 \ - --hash=sha256:036d020d79048a5d525bc63880d7a4b8d1668566b8a76daf1144c0bbe0f63472 \ - --hash=sha256:23146ed7ac4403b70bd6026402468942ceba34a6732255b9edf5b7354f68a6bb +argcomplete==3.5.3 \ + --hash=sha256:2ab2c4a215c59fd6caaff41a869480a23e8f6a5f910b266c1808037f4e375b61 \ + --hash=sha256:c12bf50eded8aebb298c7b7da7a5ff3ee24dffd9f5281867dfe1424b58c55392 # via nox +cachetools==5.5.0 \ + --hash=sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292 \ + --hash=sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a + # via google-auth +certifi==2024.12.14 \ + --hash=sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56 \ + --hash=sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db + # via requests +charset-normalizer==3.4.1 \ + --hash=sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537 \ + --hash=sha256:01732659ba9b5b873fc117534143e4feefecf3b2078b0a6a2e925271bb6f4cfa \ + --hash=sha256:01ad647cdd609225c5350561d084b42ddf732f4eeefe6e678765636791e78b9a \ + --hash=sha256:04432ad9479fa40ec0f387795ddad4437a2b50417c69fa275e212933519ff294 \ + --hash=sha256:0907f11d019260cdc3f94fbdb23ff9125f6b5d1039b76003b5b0ac9d6a6c9d5b \ + --hash=sha256:0924e81d3d5e70f8126529951dac65c1010cdf117bb75eb02dd12339b57749dd \ + --hash=sha256:09b26ae6b1abf0d27570633b2b078a2a20419c99d66fb2823173d73f188ce601 \ + --hash=sha256:09b5e6733cbd160dcc09589227187e242a30a49ca5cefa5a7edd3f9d19ed53fd \ + --hash=sha256:0af291f4fe114be0280cdd29d533696a77b5b49cfde5467176ecab32353395c4 \ + --hash=sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d \ + --hash=sha256:1a2bc9f351a75ef49d664206d51f8e5ede9da246602dc2d2726837620ea034b2 \ + --hash=sha256:22e14b5d70560b8dd51ec22863f370d1e595ac3d024cb8ad7d308b4cd95f8313 \ + --hash=sha256:234ac59ea147c59ee4da87a0c0f098e9c8d169f4dc2a159ef720f1a61bbe27cd \ + --hash=sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa \ + --hash=sha256:237bdbe6159cff53b4f24f397d43c6336c6b0b42affbe857970cefbb620911c8 \ + --hash=sha256:28bf57629c75e810b6ae989f03c0828d64d6b26a5e205535585f96093e405ed1 \ + --hash=sha256:2967f74ad52c3b98de4c3b32e1a44e32975e008a9cd2a8cc8966d6a5218c5cb2 \ + --hash=sha256:2a75d49014d118e4198bcee5ee0a6f25856b29b12dbf7cd012791f8a6cc5c496 \ + --hash=sha256:2bdfe3ac2e1bbe5b59a1a63721eb3b95fc9b6817ae4a46debbb4e11f6232428d \ + --hash=sha256:2d074908e1aecee37a7635990b2c6d504cd4766c7bc9fc86d63f9c09af3fa11b \ + --hash=sha256:2fb9bd477fdea8684f78791a6de97a953c51831ee2981f8e4f583ff3b9d9687e \ + --hash=sha256:311f30128d7d333eebd7896965bfcfbd0065f1716ec92bd5638d7748eb6f936a \ + --hash=sha256:329ce159e82018d646c7ac45b01a430369d526569ec08516081727a20e9e4af4 \ + --hash=sha256:345b0426edd4e18138d6528aed636de7a9ed169b4aaf9d61a8c19e39d26838ca \ + --hash=sha256:363e2f92b0f0174b2f8238240a1a30142e3db7b957a5dd5689b0e75fb717cc78 \ + --hash=sha256:3a3bd0dcd373514dcec91c411ddb9632c0d7d92aed7093b8c3bbb6d69ca74408 \ + --hash=sha256:3bed14e9c89dcb10e8f3a29f9ccac4955aebe93c71ae803af79265c9ca5644c5 \ + --hash=sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3 \ + --hash=sha256:44ecbf16649486d4aebafeaa7ec4c9fed8b88101f4dd612dcaf65d5e815f837f \ + --hash=sha256:4532bff1b8421fd0a320463030c7520f56a79c9024a4e88f01c537316019005a \ + --hash=sha256:49402233c892a461407c512a19435d1ce275543138294f7ef013f0b63d5d3765 \ + --hash=sha256:4c0907b1928a36d5a998d72d64d8eaa7244989f7aaaf947500d3a800c83a3fd6 \ + --hash=sha256:4d86f7aff21ee58f26dcf5ae81a9addbd914115cdebcbb2217e4f0ed8982e146 \ + --hash=sha256:5777ee0881f9499ed0f71cc82cf873d9a0ca8af166dfa0af8ec4e675b7df48e6 \ + --hash=sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9 \ + --hash=sha256:619a609aa74ae43d90ed2e89bdd784765de0a25ca761b93e196d938b8fd1dbbd \ + --hash=sha256:6e27f48bcd0957c6d4cb9d6fa6b61d192d0b13d5ef563e5f2ae35feafc0d179c \ + --hash=sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f \ + --hash=sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545 \ + --hash=sha256:7461baadb4dc00fd9e0acbe254e3d7d2112e7f92ced2adc96e54ef6501c5f176 \ + --hash=sha256:75832c08354f595c760a804588b9357d34ec00ba1c940c15e31e96d902093770 \ + --hash=sha256:7709f51f5f7c853f0fb938bcd3bc59cdfdc5203635ffd18bf354f6967ea0f824 \ + --hash=sha256:78baa6d91634dfb69ec52a463534bc0df05dbd546209b79a3880a34487f4b84f \ + --hash=sha256:7974a0b5ecd505609e3b19742b60cee7aa2aa2fb3151bc917e6e2646d7667dcf \ + --hash=sha256:7a4f97a081603d2050bfaffdefa5b02a9ec823f8348a572e39032caa8404a487 \ + --hash=sha256:7b1bef6280950ee6c177b326508f86cad7ad4dff12454483b51d8b7d673a2c5d \ + --hash=sha256:7d053096f67cd1241601111b698f5cad775f97ab25d81567d3f59219b5f1adbd \ + --hash=sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b \ + --hash=sha256:807f52c1f798eef6cf26beb819eeb8819b1622ddfeef9d0977a8502d4db6d534 \ + --hash=sha256:80ed5e856eb7f30115aaf94e4a08114ccc8813e6ed1b5efa74f9f82e8509858f \ + --hash=sha256:8417cb1f36cc0bc7eaba8ccb0e04d55f0ee52df06df3ad55259b9a323555fc8b \ + --hash=sha256:8436c508b408b82d87dc5f62496973a1805cd46727c34440b0d29d8a2f50a6c9 \ + --hash=sha256:89149166622f4db9b4b6a449256291dc87a99ee53151c74cbd82a53c8c2f6ccd \ + --hash=sha256:8bfa33f4f2672964266e940dd22a195989ba31669bd84629f05fab3ef4e2d125 \ + --hash=sha256:8c60ca7339acd497a55b0ea5d506b2a2612afb2826560416f6894e8b5770d4a9 \ + --hash=sha256:91b36a978b5ae0ee86c394f5a54d6ef44db1de0815eb43de826d41d21e4af3de \ + --hash=sha256:955f8851919303c92343d2f66165294848d57e9bba6cf6e3625485a70a038d11 \ + --hash=sha256:97f68b8d6831127e4787ad15e6757232e14e12060bec17091b85eb1486b91d8d \ + --hash=sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35 \ + --hash=sha256:9f0b8b1c6d84c8034a44893aba5e767bf9c7a211e313a9605d9c617d7083829f \ + --hash=sha256:aabfa34badd18f1da5ec1bc2715cadc8dca465868a4e73a0173466b688f29dda \ + --hash=sha256:ab36c8eb7e454e34e60eb55ca5d241a5d18b2c6244f6827a30e451c42410b5f7 \ + --hash=sha256:b010a7a4fd316c3c484d482922d13044979e78d1861f0e0650423144c616a46a \ + --hash=sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971 \ + --hash=sha256:b7b2d86dd06bfc2ade3312a83a5c364c7ec2e3498f8734282c6c3d4b07b346b8 \ + --hash=sha256:b97e690a2118911e39b4042088092771b4ae3fc3aa86518f84b8cf6888dbdb41 \ + --hash=sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d \ + --hash=sha256:c0429126cf75e16c4f0ad00ee0eae4242dc652290f940152ca8c75c3a4b6ee8f \ + --hash=sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757 \ + --hash=sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a \ + --hash=sha256:c75cb2a3e389853835e84a2d8fb2b81a10645b503eca9bcb98df6b5a43eb8886 \ + --hash=sha256:c96836c97b1238e9c9e3fe90844c947d5afbf4f4c92762679acfe19927d81d77 \ + --hash=sha256:d7f50a1f8c450f3925cb367d011448c39239bb3eb4117c36a6d354794de4ce76 \ + --hash=sha256:d973f03c0cb71c5ed99037b870f2be986c3c05e63622c017ea9816881d2dd247 \ + --hash=sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85 \ + --hash=sha256:d9c3cdf5390dcd29aa8056d13e8e99526cda0305acc038b96b30352aff5ff2bb \ + --hash=sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7 \ + --hash=sha256:dccbe65bd2f7f7ec22c4ff99ed56faa1e9f785482b9bbd7c717e26fd723a1d1e \ + --hash=sha256:dd78cfcda14a1ef52584dbb008f7ac81c1328c0f58184bf9a84c49c605002da6 \ + --hash=sha256:e218488cd232553829be0664c2292d3af2eeeb94b32bea483cf79ac6a694e037 \ + --hash=sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1 \ + --hash=sha256:ea0d8d539afa5eb2728aa1932a988a9a7af94f18582ffae4bc10b3fbdad0626e \ + --hash=sha256:eab677309cdb30d047996b36d34caeda1dc91149e4fdca0b1a039b3f79d9a807 \ + --hash=sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407 \ + --hash=sha256:ecddf25bee22fe4fe3737a399d0d177d72bc22be6913acfab364b40bce1ba83c \ + --hash=sha256:eea6ee1db730b3483adf394ea72f808b6e18cf3cb6454b4d86e04fa8c4327a12 \ + --hash=sha256:f08ff5e948271dc7e18a35641d2f11a4cd8dfd5634f55228b691e62b37125eb3 \ + --hash=sha256:f30bf9fd9be89ecb2360c7d94a711f00c09b976258846efe40db3d05828e8089 \ + --hash=sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd \ + --hash=sha256:fc54db6c8593ef7d4b2a331b58653356cf04f67c960f584edb7c3d8c97e8f39e \ + --hash=sha256:fd4ec41f914fa74ad1b8304bbc634b3de73d2a0889bd32076342a573e0779e00 \ + --hash=sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616 + # via requests +click==8.1.8 \ + --hash=sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2 \ + --hash=sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a + # via gcp-docuploader colorlog==6.9.0 \ --hash=sha256:5906e71acd67cb07a71e779c47c4bcb45fb8c2993eebe9e5adcd6a6f1b283eff \ --hash=sha256:bfba54a1b93b94f54e1f4fe48395725a3d92fd2a4af702f6bd70946bdc0c6ac2 - # via nox + # via + # gcp-docuploader + # nox distlib==0.3.9 \ --hash=sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87 \ --hash=sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403 @@ -20,10 +128,78 @@ filelock==3.16.1 \ --hash=sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0 \ --hash=sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435 # via virtualenv +gcp-docuploader==0.6.5 \ + --hash=sha256:30221d4ac3e5a2b9c69aa52fdbef68cc3f27d0e6d0d90e220fc024584b8d2318 \ + --hash=sha256:b7458ef93f605b9d46a4bf3a8dc1755dad1f31d030c8679edf304e343b347eea + # via -r requirements.in +google-api-core==2.24.0 \ + --hash=sha256:10d82ac0fca69c82a25b3efdeefccf6f28e02ebb97925a8cce8edbfe379929d9 \ + --hash=sha256:e255640547a597a4da010876d333208ddac417d60add22b6851a0c66a831fcaf + # via + # google-cloud-core + # google-cloud-storage +google-auth==2.37.0 \ + --hash=sha256:0054623abf1f9c83492c63d3f47e77f0a544caa3d40b2d98e099a611c2dd5d00 \ + --hash=sha256:42664f18290a6be591be5329a96fe30184be1a1badb7292a7f686a9659de9ca0 + # via + # google-api-core + # google-cloud-core + # google-cloud-storage +google-cloud-core==2.4.1 \ + --hash=sha256:9b7749272a812bde58fff28868d0c5e2f585b82f37e09a1f6ed2d4d10f134073 \ + --hash=sha256:a9e6a4422b9ac5c29f79a0ede9485473338e2ce78d91f2370c01e730eab22e61 + # via google-cloud-storage +google-cloud-storage==2.19.0 \ + --hash=sha256:aeb971b5c29cf8ab98445082cbfe7b161a1f48ed275822f59ed3f1524ea54fba \ + --hash=sha256:cd05e9e7191ba6cb68934d8eb76054d9be4562aa89dbc4236feee4d7d51342b2 + # via gcp-docuploader +google-crc32c==1.6.0 \ + --hash=sha256:05e2d8c9a2f853ff116db9706b4a27350587f341eda835f46db3c0a8c8ce2f24 \ + --hash=sha256:18e311c64008f1f1379158158bb3f0c8d72635b9eb4f9545f8cf990c5668e59d \ + --hash=sha256:236c87a46cdf06384f614e9092b82c05f81bd34b80248021f729396a78e55d7e \ + --hash=sha256:35834855408429cecf495cac67ccbab802de269e948e27478b1e47dfb6465e57 \ + --hash=sha256:386122eeaaa76951a8196310432c5b0ef3b53590ef4c317ec7588ec554fec5d2 \ + --hash=sha256:40b05ab32a5067525670880eb5d169529089a26fe35dce8891127aeddc1950e8 \ + --hash=sha256:48abd62ca76a2cbe034542ed1b6aee851b6f28aaca4e6551b5599b6f3ef175cc \ + --hash=sha256:50cf2a96da226dcbff8671233ecf37bf6e95de98b2a2ebadbfdf455e6d05df42 \ + --hash=sha256:51c4f54dd8c6dfeb58d1df5e4f7f97df8abf17a36626a217f169893d1d7f3e9f \ + --hash=sha256:5bcc90b34df28a4b38653c36bb5ada35671ad105c99cfe915fb5bed7ad6924aa \ + --hash=sha256:62f6d4a29fea082ac4a3c9be5e415218255cf11684ac6ef5488eea0c9132689b \ + --hash=sha256:6eceb6ad197656a1ff49ebfbbfa870678c75be4344feb35ac1edf694309413dc \ + --hash=sha256:7aec8e88a3583515f9e0957fe4f5f6d8d4997e36d0f61624e70469771584c760 \ + --hash=sha256:91ca8145b060679ec9176e6de4f89b07363d6805bd4760631ef254905503598d \ + --hash=sha256:a184243544811e4a50d345838a883733461e67578959ac59964e43cca2c791e7 \ + --hash=sha256:a9e4b426c3702f3cd23b933436487eb34e01e00327fac20c9aebb68ccf34117d \ + --hash=sha256:bb0966e1c50d0ef5bc743312cc730b533491d60585a9a08f897274e57c3f70e0 \ + --hash=sha256:bb8b3c75bd157010459b15222c3fd30577042a7060e29d42dabce449c087f2b3 \ + --hash=sha256:bd5e7d2445d1a958c266bfa5d04c39932dc54093fa391736dbfdb0f1929c1fb3 \ + --hash=sha256:c87d98c7c4a69066fd31701c4e10d178a648c2cac3452e62c6b24dc51f9fcc00 \ + --hash=sha256:d2952396dc604544ea7476b33fe87faedc24d666fb0c2d5ac971a2b9576ab871 \ + --hash=sha256:d8797406499f28b5ef791f339594b0b5fdedf54e203b5066675c406ba69d705c \ + --hash=sha256:d9e9913f7bd69e093b81da4535ce27af842e7bf371cde42d1ae9e9bd382dc0e9 \ + --hash=sha256:e2806553238cd076f0a55bddab37a532b53580e699ed8e5606d0de1f856b5205 \ + --hash=sha256:ebab974b1687509e5c973b5c4b8b146683e101e102e17a86bd196ecaa4d099fc \ + --hash=sha256:ed767bf4ba90104c1216b68111613f0d5926fb3780660ea1198fc469af410e9d \ + --hash=sha256:f7a1fc29803712f80879b0806cb83ab24ce62fc8daf0569f2204a0cfd7f68ed4 + # via + # google-cloud-storage + # google-resumable-media +google-resumable-media==2.7.2 \ + --hash=sha256:3ce7551e9fe6d99e9a126101d2536612bb73486721951e9562fee0f90c6ababa \ + --hash=sha256:5280aed4629f2b60b847b0d42f9857fd4935c11af266744df33d8074cae92fe0 + # via google-cloud-storage +googleapis-common-protos==1.66.0 \ + --hash=sha256:c3e7b33d15fdca5374cc0a7346dd92ffa847425cc4ea941d970f13680052ec8c \ + --hash=sha256:d7abcd75fabb2e0ec9f74466401f6c119a0b498e27370e9be4c94cb7e382b8ed + # via google-api-core +idna==3.10 \ + --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ + --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 + # via requests nox==2024.10.9 \ --hash=sha256:1d36f309a0a2a853e9bccb76bbef6bb118ba92fa92674d15604ca99adeb29eab \ --hash=sha256:7aa9dc8d1c27e9f45ab046ffd1c3b2c4f7c91755304769df231308849ebded95 - # via -r synthtool/gcp/templates/python_library/.kokoro/docker/docs/requirements.in + # via -r requirements.in packaging==24.2 \ --hash=sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759 \ --hash=sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f @@ -32,6 +208,51 @@ platformdirs==4.3.6 \ --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \ --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb # via virtualenv +proto-plus==1.25.0 \ + --hash=sha256:c91fc4a65074ade8e458e95ef8bac34d4008daa7cce4a12d6707066fca648961 \ + --hash=sha256:fbb17f57f7bd05a68b7707e745e26528b0b3c34e378db91eef93912c54982d91 + # via google-api-core +protobuf==5.29.3 \ + --hash=sha256:0a18ed4a24198528f2333802eb075e59dea9d679ab7a6c5efb017a59004d849f \ + --hash=sha256:0eb32bfa5219fc8d4111803e9a690658aa2e6366384fd0851064b963b6d1f2a7 \ + --hash=sha256:3ea51771449e1035f26069c4c7fd51fba990d07bc55ba80701c78f886bf9c888 \ + --hash=sha256:5da0f41edaf117bde316404bad1a486cb4ededf8e4a54891296f648e8e076620 \ + --hash=sha256:6ce8cc3389a20693bfde6c6562e03474c40851b44975c9b2bf6df7d8c4f864da \ + --hash=sha256:84a57163a0ccef3f96e4b6a20516cedcf5bb3a95a657131c5c3ac62200d23252 \ + --hash=sha256:a4fa6f80816a9a0678429e84973f2f98cbc218cca434abe8db2ad0bffc98503a \ + --hash=sha256:a8434404bbf139aa9e1300dbf989667a83d42ddda9153d8ab76e0d5dcaca484e \ + --hash=sha256:b89c115d877892a512f79a8114564fb435943b59067615894c3b13cd3e1fa107 \ + --hash=sha256:c027e08a08be10b67c06bf2370b99c811c466398c357e615ca88c91c07f0910f \ + --hash=sha256:daaf63f70f25e8689c072cfad4334ca0ac1d1e05a92fc15c54eb9cf23c3efd84 + # via + # gcp-docuploader + # google-api-core + # googleapis-common-protos + # proto-plus +pyasn1==0.6.1 \ + --hash=sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629 \ + --hash=sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.1 \ + --hash=sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd \ + --hash=sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c + # via google-auth +requests==2.32.3 \ + --hash=sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760 \ + --hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6 + # via + # google-api-core + # google-cloud-storage +rsa==4.9 \ + --hash=sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7 \ + --hash=sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21 + # via google-auth +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via gcp-docuploader tomli==2.2.1 \ --hash=sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6 \ --hash=sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd \ @@ -66,7 +287,11 @@ tomli==2.2.1 \ --hash=sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a \ --hash=sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7 # via nox -virtualenv==20.28.0 \ - --hash=sha256:23eae1b4516ecd610481eda647f3a7c09aea295055337331bb4e6892ecce47b0 \ - --hash=sha256:2c9c3262bb8e7b87ea801d715fae4495e6032450c71d2309be9550e7364049aa +urllib3==2.3.0 \ + --hash=sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df \ + --hash=sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d + # via requests +virtualenv==20.28.1 \ + --hash=sha256:412773c85d4dab0409b83ec36f7a6499e72eaf08c80e81e9576bca61831c71cb \ + --hash=sha256:5d34ab240fdb5d21549b76f9e8ff3af28252f5499fb6d6f031adac4e5a8c5329 # via nox diff --git a/.kokoro/publish-docs.sh b/.kokoro/publish-docs.sh index 233205d58..4ed4aaf13 100755 --- a/.kokoro/publish-docs.sh +++ b/.kokoro/publish-docs.sh @@ -20,10 +20,6 @@ export PYTHONUNBUFFERED=1 export PATH="${HOME}/.local/bin:${PATH}" -# Install nox -python3.10 -m pip install --require-hashes -r .kokoro/requirements.txt -python3.10 -m nox --version - # build docs nox -s docs diff --git a/.kokoro/requirements.txt b/.kokoro/requirements.txt index 9622baf0b..14466eb39 100644 --- a/.kokoro/requirements.txt +++ b/.kokoro/requirements.txt @@ -517,9 +517,9 @@ urllib3==2.2.2 \ # via # requests # twine -virtualenv==20.26.3 \ - --hash=sha256:4c43a2a236279d9ea36a0d76f98d84bd6ca94ac4e0f4a3b9d46d05e10fea542a \ - --hash=sha256:8cc4a31139e796e9a7de2cd5cf2489de1217193116a8fd42328f1bd65f434589 +virtualenv==20.26.6 \ + --hash=sha256:280aede09a2a5c317e409a00102e7077c6432c5a38f0ef938e643805a7ad2c48 \ + --hash=sha256:7345cc5b25405607a624d8418154577459c3e0277f5466dd79c49d5e492995f2 # via nox wheel==0.43.0 \ --hash=sha256:465ef92c69fa5c5da2d1cf8ac40559a8c940886afcef87dcf14b9470862f1d85 \ diff --git a/.repo-metadata.json b/.repo-metadata.json index 9e537d52f..5d5e49c84 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -12,7 +12,7 @@ "api_id": "storage.googleapis.com", "requires_billing": true, "default_version": "", - "codeowner_team": "@googleapis/cloud-storage-dpe", + "codeowner_team": "@googleapis/gcs-sdk-team", "api_shortname": "storage", "api_description": "is a durable and highly available object storage service. Google Cloud Storage is almost infinitely scalable and guarantees consistency: when a write succeeds, the latest copy of the object will be returned to any GET, globally." } diff --git a/CHANGELOG.md b/CHANGELOG.md index dcf58ac2a..fefb84f50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ [1]: https://pypi.org/project/google-cloud-storage/#history +## [3.1.0](https://github.com/googleapis/python-storage/compare/v3.0.0...v3.1.0) (2025-02-27) + + +### Features + +* Add api_key argument to Client constructor ([#1441](https://github.com/googleapis/python-storage/issues/1441)) ([c869e15](https://github.com/googleapis/python-storage/commit/c869e15ec535a0aa50029d30b6a3ce64ff119b5f)) +* Add Bucket.move_blob() for HNS-enabled buckets ([#1431](https://github.com/googleapis/python-storage/issues/1431)) ([24c000f](https://github.com/googleapis/python-storage/commit/24c000fb7b9f576e6d6c6ec5733f3971fe133655)) + ## [3.0.0](https://github.com/googleapis/python-storage/compare/v2.19.0...v3.0.0) (2025-01-28) diff --git a/google/cloud/storage/acl.py b/google/cloud/storage/acl.py index 1384a5075..d70839e1b 100644 --- a/google/cloud/storage/acl.py +++ b/google/cloud/storage/acl.py @@ -360,7 +360,6 @@ def _require_client(self, client): client = self.client return client - @create_trace_span(name="Storage.ACL.reload") def reload(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): """Reload the ACL data from Cloud Storage. @@ -379,25 +378,26 @@ def reload(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): :param retry: (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ - path = self.reload_path - client = self._require_client(client) - query_params = {} + with create_trace_span(name="Storage.ACL.reload"): + path = self.reload_path + client = self._require_client(client) + query_params = {} - if self.user_project is not None: - query_params["userProject"] = self.user_project + if self.user_project is not None: + query_params["userProject"] = self.user_project - self.entities.clear() + self.entities.clear() - found = client._get_resource( - path, - query_params=query_params, - timeout=timeout, - retry=retry, - ) - self.loaded = True + found = client._get_resource( + path, + query_params=query_params, + timeout=timeout, + retry=retry, + ) + self.loaded = True - for entry in found.get("items", ()): - self.add_entity(self.entity_from_dict(entry)) + for entry in found.get("items", ()): + self.add_entity(self.entity_from_dict(entry)) def _save( self, @@ -486,7 +486,6 @@ def _save( self.loaded = True - @create_trace_span(name="Storage.ACL.save") def save( self, acl=None, @@ -536,26 +535,26 @@ def save( :param retry: (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ - if acl is None: - acl = self - save_to_backend = acl.loaded - else: - save_to_backend = True + with create_trace_span(name="Storage.ACL.save"): + if acl is None: + acl = self + save_to_backend = acl.loaded + else: + save_to_backend = True + + if save_to_backend: + self._save( + acl, + None, + client, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + retry=retry, + ) - if save_to_backend: - self._save( - acl, - None, - client, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - retry=retry, - ) - - @create_trace_span(name="Storage.ACL.savePredefined") def save_predefined( self, predefined, @@ -608,20 +607,20 @@ def save_predefined( :param retry: (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ - predefined = self.validate_predefined(predefined) - self._save( - None, - predefined, - client, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - retry=retry, - ) + with create_trace_span(name="Storage.ACL.savePredefined"): + predefined = self.validate_predefined(predefined) + self._save( + None, + predefined, + client, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + retry=retry, + ) - @create_trace_span(name="Storage.ACL.clear") def clear( self, client=None, @@ -671,16 +670,17 @@ def clear( :param retry: (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ - self.save( - [], - client=client, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - retry=retry, - ) + with create_trace_span(name="Storage.ACL.clear"): + self.save( + [], + client=client, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + retry=retry, + ) class BucketACL(ACL): diff --git a/google/cloud/storage/blob.py b/google/cloud/storage/blob.py index 3cda582ca..0d0e8ee80 100644 --- a/google/cloud/storage/blob.py +++ b/google/cloud/storage/blob.py @@ -675,7 +675,6 @@ def generate_signed_url( universe_domain=universe_domain, ) - @create_trace_span(name="Storage.Blob.exists") def exists( self, client=None, @@ -742,46 +741,48 @@ def exists( :rtype: bool :returns: True if the blob exists in Cloud Storage. """ - client = self._require_client(client) - # We only need the status code (200 or not) so we seek to - # minimize the returned payload. - query_params = self._query_params - query_params["fields"] = "name" - if soft_deleted is not None: - query_params["softDeleted"] = soft_deleted + with create_trace_span(name="Storage.Blob.exists"): + client = self._require_client(client) + # We only need the status code (200 or not) so we seek to + # minimize the returned payload. + query_params = self._query_params + query_params["fields"] = "name" + if soft_deleted is not None: + query_params["softDeleted"] = soft_deleted + + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) - _add_generation_match_parameters( - query_params, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) + headers = {} + _add_etag_match_headers( + headers, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + ) - headers = {} - _add_etag_match_headers( - headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match - ) + try: + # We intentionally pass `_target_object=None` since fields=name + # would limit the local properties. + client._get_resource( + self.path, + query_params=query_params, + headers=headers, + timeout=timeout, + retry=retry, + _target_object=None, + ) + except NotFound: + # NOTE: This will not fail immediately in a batch. However, when + # Batch.finish() is called, the resulting `NotFound` will be + # raised. + return False + return True - try: - # We intentionally pass `_target_object=None` since fields=name - # would limit the local properties. - client._get_resource( - self.path, - query_params=query_params, - headers=headers, - timeout=timeout, - retry=retry, - _target_object=None, - ) - except NotFound: - # NOTE: This will not fail immediately in a batch. However, when - # Batch.finish() is called, the resulting `NotFound` will be - # raised. - return False - return True - - @create_trace_span(name="Storage.Blob.delete") def delete( self, client=None, @@ -844,17 +845,18 @@ def delete( (propagated from :meth:`google.cloud.storage.bucket.Bucket.delete_blob`). """ - self.bucket.delete_blob( - self.name, - client=client, - generation=self.generation, - timeout=timeout, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - ) + with create_trace_span(name="Storage.Blob.delete"): + self.bucket.delete_blob( + self.name, + client=client, + generation=self.generation, + timeout=timeout, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, + ) def _get_transport(self, client): """Return the client's transport. @@ -1109,7 +1111,6 @@ def _do_download( while not download.finished: download.consume_next_chunk(transport, timeout=timeout) - @create_trace_span(name="Storage.Blob.downloadToFile") def download_to_file( self, file_obj, @@ -1223,23 +1224,23 @@ def download_to_file( :raises: :class:`google.cloud.exceptions.NotFound` """ - - self._prep_and_do_download( - file_obj, - client=client, - start=start, - end=end, - raw_download=raw_download, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - checksum=checksum, - retry=retry, - ) + with create_trace_span(name="Storage.Blob.downloadToFile"): + self._prep_and_do_download( + file_obj, + client=client, + start=start, + end=end, + raw_download=raw_download, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + checksum=checksum, + retry=retry, + ) def _handle_filename_and_download(self, filename, *args, **kwargs): """Download the contents of this blob into a named file. @@ -1268,7 +1269,6 @@ def _handle_filename_and_download(self, filename, *args, **kwargs): mtime = updated.timestamp() os.utime(file_obj.name, (mtime, mtime)) - @create_trace_span(name="Storage.Blob.downloadToFilename") def download_to_filename( self, filename, @@ -1372,25 +1372,24 @@ def download_to_filename( :raises: :class:`google.cloud.exceptions.NotFound` """ + with create_trace_span(name="Storage.Blob.downloadToFilename"): + self._handle_filename_and_download( + filename, + client=client, + start=start, + end=end, + raw_download=raw_download, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + checksum=checksum, + retry=retry, + ) - self._handle_filename_and_download( - filename, - client=client, - start=start, - end=end, - raw_download=raw_download, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - checksum=checksum, - retry=retry, - ) - - @create_trace_span(name="Storage.Blob.downloadAsBytes") def download_as_bytes( self, client=None, @@ -1490,28 +1489,27 @@ def download_as_bytes( :raises: :class:`google.cloud.exceptions.NotFound` """ + with create_trace_span(name="Storage.Blob.downloadAsBytes"): + string_buffer = BytesIO() - string_buffer = BytesIO() - - self._prep_and_do_download( - string_buffer, - client=client, - start=start, - end=end, - raw_download=raw_download, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - checksum=checksum, - retry=retry, - ) - return string_buffer.getvalue() + self._prep_and_do_download( + string_buffer, + client=client, + start=start, + end=end, + raw_download=raw_download, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + checksum=checksum, + retry=retry, + ) + return string_buffer.getvalue() - @create_trace_span(name="Storage.Blob.downloadAsString") def download_as_string( self, client=None, @@ -1604,22 +1602,22 @@ def download_as_string( warnings.warn( _DOWNLOAD_AS_STRING_DEPRECATED, PendingDeprecationWarning, stacklevel=2 ) - return self.download_as_bytes( - client=client, - start=start, - end=end, - raw_download=raw_download, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - retry=retry, - ) + with create_trace_span(name="Storage.Blob.downloadAsString"): + return self.download_as_bytes( + client=client, + start=start, + end=end, + raw_download=raw_download, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + retry=retry, + ) - @create_trace_span(name="Storage.Blob.downloadAsText") def download_as_text( self, client=None, @@ -1710,31 +1708,32 @@ def download_as_text( :rtype: text :returns: The data stored in this blob, decoded to text. """ - data = self.download_as_bytes( - client=client, - start=start, - end=end, - raw_download=raw_download, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - retry=retry, - ) + with create_trace_span(name="Storage.Blob.downloadAsText"): + data = self.download_as_bytes( + client=client, + start=start, + end=end, + raw_download=raw_download, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + retry=retry, + ) - if encoding is not None: - return data.decode(encoding) + if encoding is not None: + return data.decode(encoding) - if self.content_type is not None: - msg = HeaderParser().parsestr("Content-Type: " + self.content_type) - params = dict(msg.get_params()[1:]) - if "charset" in params: - return data.decode(params["charset"]) + if self.content_type is not None: + msg = HeaderParser().parsestr("Content-Type: " + self.content_type) + params = dict(msg.get_params()[1:]) + if "charset" in params: + return data.decode(params["charset"]) - return data.decode("utf-8") + return data.decode("utf-8") def _get_content_type(self, content_type, filename=None): """Determine the content type from the current object. @@ -2655,7 +2654,6 @@ def _prep_and_do_upload( except InvalidResponse as exc: _raise_from_invalid_response(exc) - @create_trace_span(name="Storage.Blob.uploadFromFile") def upload_from_file( self, file_obj, @@ -2782,21 +2780,22 @@ def upload_from_file( :raises: :class:`~google.cloud.exceptions.GoogleCloudError` if the upload response returns an error status. """ - self._prep_and_do_upload( - file_obj, - rewind=rewind, - size=size, - content_type=content_type, - client=client, - predefined_acl=predefined_acl, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - checksum=checksum, - retry=retry, - ) + with create_trace_span(name="Storage.Blob.uploadFromFile"): + self._prep_and_do_upload( + file_obj, + rewind=rewind, + size=size, + content_type=content_type, + client=client, + predefined_acl=predefined_acl, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + checksum=checksum, + retry=retry, + ) def _handle_filename_and_upload(self, filename, content_type=None, *args, **kwargs): """Upload this blob's contents from the content of a named file. @@ -2822,7 +2821,6 @@ def _handle_filename_and_upload(self, filename, content_type=None, *args, **kwar **kwargs, ) - @create_trace_span(name="Storage.Blob.uploadFromFilename") def upload_from_filename( self, filename, @@ -2931,22 +2929,21 @@ def upload_from_filename( (google.cloud.storage.retry) for information on retry types and how to configure them. """ + with create_trace_span(name="Storage.Blob.uploadFromFilename"): + self._handle_filename_and_upload( + filename, + content_type=content_type, + client=client, + predefined_acl=predefined_acl, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + checksum=checksum, + retry=retry, + ) - self._handle_filename_and_upload( - filename, - content_type=content_type, - client=client, - predefined_acl=predefined_acl, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - checksum=checksum, - retry=retry, - ) - - @create_trace_span(name="Storage.Blob.uploadFromString") def upload_from_string( self, data, @@ -3047,24 +3044,24 @@ def upload_from_string( (google.cloud.storage.retry) for information on retry types and how to configure them. """ - data = _to_bytes(data, encoding="utf-8") - string_buffer = BytesIO(data) - self.upload_from_file( - file_obj=string_buffer, - size=len(data), - content_type=content_type, - client=client, - predefined_acl=predefined_acl, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - checksum=checksum, - retry=retry, - ) + with create_trace_span(name="Storage.Blob.uploadFromString"): + data = _to_bytes(data, encoding="utf-8") + string_buffer = BytesIO(data) + self.upload_from_file( + file_obj=string_buffer, + size=len(data), + content_type=content_type, + client=client, + predefined_acl=predefined_acl, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + checksum=checksum, + retry=retry, + ) - @create_trace_span(name="Storage.Blob.createResumableUploadSession") def create_resumable_upload_session( self, content_type=None, @@ -3198,52 +3195,53 @@ def create_resumable_upload_session( :raises: :class:`google.cloud.exceptions.GoogleCloudError` if the session creation response returns an error status. """ + with create_trace_span(name="Storage.Blob.createResumableUploadSession"): + # Handle ConditionalRetryPolicy. + if isinstance(retry, ConditionalRetryPolicy): + # Conditional retries are designed for non-media calls, which change + # arguments into query_params dictionaries. Media operations work + # differently, so here we make a "fake" query_params to feed to the + # ConditionalRetryPolicy. + query_params = { + "ifGenerationMatch": if_generation_match, + "ifMetagenerationMatch": if_metageneration_match, + } + retry = retry.get_retry_policy_if_conditions_met( + query_params=query_params + ) - # Handle ConditionalRetryPolicy. - if isinstance(retry, ConditionalRetryPolicy): - # Conditional retries are designed for non-media calls, which change - # arguments into query_params dictionaries. Media operations work - # differently, so here we make a "fake" query_params to feed to the - # ConditionalRetryPolicy. - query_params = { - "ifGenerationMatch": if_generation_match, - "ifMetagenerationMatch": if_metageneration_match, - } - retry = retry.get_retry_policy_if_conditions_met(query_params=query_params) - - extra_headers = {} - if origin is not None: - # This header is specifically for client-side uploads, it - # determines the origins allowed for CORS. - extra_headers["Origin"] = origin - - try: - fake_stream = BytesIO(b"") - # Send a fake the chunk size which we **know** will be acceptable - # to the `ResumableUpload` constructor. The chunk size only - # matters when **sending** bytes to an upload. - upload, _ = self._initiate_resumable_upload( - client, - fake_stream, - content_type, - size, - predefined_acl=predefined_acl, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - extra_headers=extra_headers, - chunk_size=self._CHUNK_SIZE_MULTIPLE, - timeout=timeout, - checksum=checksum, - retry=retry, - ) + extra_headers = {} + if origin is not None: + # This header is specifically for client-side uploads, it + # determines the origins allowed for CORS. + extra_headers["Origin"] = origin + + try: + fake_stream = BytesIO(b"") + # Send a fake the chunk size which we **know** will be acceptable + # to the `ResumableUpload` constructor. The chunk size only + # matters when **sending** bytes to an upload. + upload, _ = self._initiate_resumable_upload( + client, + fake_stream, + content_type, + size, + predefined_acl=predefined_acl, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + extra_headers=extra_headers, + chunk_size=self._CHUNK_SIZE_MULTIPLE, + timeout=timeout, + checksum=checksum, + retry=retry, + ) - return upload.resumable_url - except InvalidResponse as exc: - _raise_from_invalid_response(exc) + return upload.resumable_url + except InvalidResponse as exc: + _raise_from_invalid_response(exc) - @create_trace_span(name="Storage.Blob.getIamPolicy") def get_iam_policy( self, client=None, @@ -3293,26 +3291,26 @@ def get_iam_policy( :returns: the policy instance, based on the resource returned from the ``getIamPolicy`` API request. """ - client = self._require_client(client) + with create_trace_span(name="Storage.Blob.getIamPolicy"): + client = self._require_client(client) - query_params = {} + query_params = {} - if self.user_project is not None: - query_params["userProject"] = self.user_project + if self.user_project is not None: + query_params["userProject"] = self.user_project - if requested_policy_version is not None: - query_params["optionsRequestedPolicyVersion"] = requested_policy_version + if requested_policy_version is not None: + query_params["optionsRequestedPolicyVersion"] = requested_policy_version - info = client._get_resource( - f"{self.path}/iam", - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=None, - ) - return Policy.from_api_repr(info) + info = client._get_resource( + f"{self.path}/iam", + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=None, + ) + return Policy.from_api_repr(info) - @create_trace_span(name="Storage.Blob.setIamPolicy") def set_iam_policy( self, policy, @@ -3354,27 +3352,27 @@ def set_iam_policy( :returns: the policy instance, based on the resource returned from the ``setIamPolicy`` API request. """ - client = self._require_client(client) + with create_trace_span(name="Storage.Blob.setIamPolicy"): + client = self._require_client(client) - query_params = {} + query_params = {} - if self.user_project is not None: - query_params["userProject"] = self.user_project - - path = f"{self.path}/iam" - resource = policy.to_api_repr() - resource["resourceId"] = self.path - info = client._put_resource( - path, - resource, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=None, - ) - return Policy.from_api_repr(info) + if self.user_project is not None: + query_params["userProject"] = self.user_project + + path = f"{self.path}/iam" + resource = policy.to_api_repr() + resource["resourceId"] = self.path + info = client._put_resource( + path, + resource, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=None, + ) + return Policy.from_api_repr(info) - @create_trace_span(name="Storage.Blob.testIamPermissions") def test_iam_permissions( self, permissions, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY ): @@ -3412,24 +3410,24 @@ def test_iam_permissions( :returns: the permissions returned by the ``testIamPermissions`` API request. """ - client = self._require_client(client) - query_params = {"permissions": permissions} + with create_trace_span(name="Storage.Blob.testIamPermissions"): + client = self._require_client(client) + query_params = {"permissions": permissions} - if self.user_project is not None: - query_params["userProject"] = self.user_project + if self.user_project is not None: + query_params["userProject"] = self.user_project - path = f"{self.path}/iam/testPermissions" - resp = client._get_resource( - path, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=None, - ) + path = f"{self.path}/iam/testPermissions" + resp = client._get_resource( + path, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=None, + ) - return resp.get("permissions", []) + return resp.get("permissions", []) - @create_trace_span(name="Storage.Blob.makePublic") def make_public( self, client=None, @@ -3472,18 +3470,18 @@ def make_public( :param retry: (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ - self.acl.all().grant_read() - self.acl.save( - client=client, - timeout=timeout, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - ) + with create_trace_span(name="Storage.Blob.makePublic"): + self.acl.all().grant_read() + self.acl.save( + client=client, + timeout=timeout, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, + ) - @create_trace_span(name="Storage.Blob.makePrivate") def make_private( self, client=None, @@ -3526,18 +3524,18 @@ def make_private( :param retry: (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ - self.acl.all().revoke_read() - self.acl.save( - client=client, - timeout=timeout, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - ) + with create_trace_span(name="Storage.Blob.makePrivate"): + self.acl.all().revoke_read() + self.acl.save( + client=client, + timeout=timeout, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, + ) - @create_trace_span(name="Storage.Blob.compose") def compose( self, sources, @@ -3607,77 +3605,77 @@ def compose( to enable retries regardless of generation precondition setting. See [Configuring Retries](https://cloud.google.com/python/docs/reference/storage/latest/retry_timeout). """ - sources_len = len(sources) - client = self._require_client(client) - query_params = {} - - if isinstance(if_generation_match, list): - warnings.warn( - _COMPOSE_IF_GENERATION_LIST_DEPRECATED, - DeprecationWarning, - stacklevel=2, - ) + with create_trace_span(name="Storage.Blob.compose"): + sources_len = len(sources) + client = self._require_client(client) + query_params = {} - if if_source_generation_match is not None: - raise ValueError( - _COMPOSE_IF_GENERATION_LIST_AND_IF_SOURCE_GENERATION_ERROR + if isinstance(if_generation_match, list): + warnings.warn( + _COMPOSE_IF_GENERATION_LIST_DEPRECATED, + DeprecationWarning, + stacklevel=2, ) - if_source_generation_match = if_generation_match - if_generation_match = None + if if_source_generation_match is not None: + raise ValueError( + _COMPOSE_IF_GENERATION_LIST_AND_IF_SOURCE_GENERATION_ERROR + ) - if isinstance(if_metageneration_match, list): - warnings.warn( - _COMPOSE_IF_METAGENERATION_LIST_DEPRECATED, - DeprecationWarning, - stacklevel=2, - ) + if_source_generation_match = if_generation_match + if_generation_match = None - if_metageneration_match = None + if isinstance(if_metageneration_match, list): + warnings.warn( + _COMPOSE_IF_METAGENERATION_LIST_DEPRECATED, + DeprecationWarning, + stacklevel=2, + ) - if if_source_generation_match is None: - if_source_generation_match = [None] * sources_len - if len(if_source_generation_match) != sources_len: - raise ValueError(_COMPOSE_IF_SOURCE_GENERATION_MISMATCH_ERROR) + if_metageneration_match = None - source_objects = [] - for source, source_generation in zip(sources, if_source_generation_match): - source_object = {"name": source.name, "generation": source.generation} + if if_source_generation_match is None: + if_source_generation_match = [None] * sources_len + if len(if_source_generation_match) != sources_len: + raise ValueError(_COMPOSE_IF_SOURCE_GENERATION_MISMATCH_ERROR) - preconditions = {} - if source_generation is not None: - preconditions["ifGenerationMatch"] = source_generation + source_objects = [] + for source, source_generation in zip(sources, if_source_generation_match): + source_object = {"name": source.name, "generation": source.generation} - if preconditions: - source_object["objectPreconditions"] = preconditions + preconditions = {} + if source_generation is not None: + preconditions["ifGenerationMatch"] = source_generation - source_objects.append(source_object) + if preconditions: + source_object["objectPreconditions"] = preconditions - request = { - "sourceObjects": source_objects, - "destination": self._properties.copy(), - } + source_objects.append(source_object) - if self.user_project is not None: - query_params["userProject"] = self.user_project + request = { + "sourceObjects": source_objects, + "destination": self._properties.copy(), + } - _add_generation_match_parameters( - query_params, - if_generation_match=if_generation_match, - if_metageneration_match=if_metageneration_match, - ) + if self.user_project is not None: + query_params["userProject"] = self.user_project - api_response = client._post_resource( - f"{self.path}/compose", - request, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=self, - ) - self._set_properties(api_response) + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_metageneration_match=if_metageneration_match, + ) + + api_response = client._post_resource( + f"{self.path}/compose", + request, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=self, + ) + self._set_properties(api_response) - @create_trace_span(name="Storage.Blob.rewrite") def rewrite( self, source, @@ -3783,66 +3781,66 @@ def rewrite( and ``total_bytes`` is the total number of bytes to be rewritten. """ - client = self._require_client(client) - headers = _get_encryption_headers(self._encryption_key) - headers.update(_get_encryption_headers(source._encryption_key, source=True)) - - query_params = self._query_params - if "generation" in query_params: - del query_params["generation"] - - if token: - query_params["rewriteToken"] = token - - if source.generation: - query_params["sourceGeneration"] = source.generation - - # When a Customer Managed Encryption Key is used to encrypt Cloud Storage object - # at rest, object resource metadata will store the version of the Key Management - # Service cryptographic material. If a Blob instance with KMS Key metadata set is - # used to rewrite the object, then the existing kmsKeyName version - # value can't be used in the rewrite request and the client instead ignores it. - if ( - self.kms_key_name is not None - and "cryptoKeyVersions" not in self.kms_key_name - ): - query_params["destinationKmsKeyName"] = self.kms_key_name + with create_trace_span(name="Storage.Blob.rewrite"): + client = self._require_client(client) + headers = _get_encryption_headers(self._encryption_key) + headers.update(_get_encryption_headers(source._encryption_key, source=True)) + + query_params = self._query_params + if "generation" in query_params: + del query_params["generation"] + + if token: + query_params["rewriteToken"] = token + + if source.generation: + query_params["sourceGeneration"] = source.generation + + # When a Customer Managed Encryption Key is used to encrypt Cloud Storage object + # at rest, object resource metadata will store the version of the Key Management + # Service cryptographic material. If a Blob instance with KMS Key metadata set is + # used to rewrite the object, then the existing kmsKeyName version + # value can't be used in the rewrite request and the client instead ignores it. + if ( + self.kms_key_name is not None + and "cryptoKeyVersions" not in self.kms_key_name + ): + query_params["destinationKmsKeyName"] = self.kms_key_name - _add_generation_match_parameters( - query_params, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - if_source_generation_match=if_source_generation_match, - if_source_generation_not_match=if_source_generation_not_match, - if_source_metageneration_match=if_source_metageneration_match, - if_source_metageneration_not_match=if_source_metageneration_not_match, - ) + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + if_source_generation_match=if_source_generation_match, + if_source_generation_not_match=if_source_generation_not_match, + if_source_metageneration_match=if_source_metageneration_match, + if_source_metageneration_not_match=if_source_metageneration_not_match, + ) - path = f"{source.path}/rewriteTo{self.path}" - api_response = client._post_resource( - path, - self._properties, - query_params=query_params, - headers=headers, - timeout=timeout, - retry=retry, - _target_object=self, - ) - rewritten = int(api_response["totalBytesRewritten"]) - size = int(api_response["objectSize"]) + path = f"{source.path}/rewriteTo{self.path}" + api_response = client._post_resource( + path, + self._properties, + query_params=query_params, + headers=headers, + timeout=timeout, + retry=retry, + _target_object=self, + ) + rewritten = int(api_response["totalBytesRewritten"]) + size = int(api_response["objectSize"]) - # The resource key is set if and only if the API response is - # completely done. Additionally, there is no rewrite token to return - # in this case. - if api_response["done"]: - self._set_properties(api_response["resource"]) - return None, rewritten, size + # The resource key is set if and only if the API response is + # completely done. Additionally, there is no rewrite token to return + # in this case. + if api_response["done"]: + self._set_properties(api_response["resource"]) + return None, rewritten, size - return api_response["rewriteToken"], rewritten, size + return api_response["rewriteToken"], rewritten, size - @create_trace_span(name="Storage.Blob.updateStorageClass") def update_storage_class( self, new_class, @@ -3943,27 +3941,13 @@ def update_storage_class( to enable retries regardless of generation precondition setting. See [Configuring Retries](https://cloud.google.com/python/docs/reference/storage/latest/retry_timeout). """ - # Update current blob's storage class prior to rewrite - self._patch_property("storageClass", new_class) + with create_trace_span(name="Storage.Blob.updateStorageClass"): + # Update current blob's storage class prior to rewrite + self._patch_property("storageClass", new_class) - # Execute consecutive rewrite operations until operation is done - token, _, _ = self.rewrite( - self, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - if_source_generation_match=if_source_generation_match, - if_source_generation_not_match=if_source_generation_not_match, - if_source_metageneration_match=if_source_metageneration_match, - if_source_metageneration_not_match=if_source_metageneration_not_match, - timeout=timeout, - retry=retry, - ) - while token is not None: + # Execute consecutive rewrite operations until operation is done token, _, _ = self.rewrite( self, - token=token, if_generation_match=if_generation_match, if_generation_not_match=if_generation_not_match, if_metageneration_match=if_metageneration_match, @@ -3975,8 +3959,22 @@ def update_storage_class( timeout=timeout, retry=retry, ) + while token is not None: + token, _, _ = self.rewrite( + self, + token=token, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + if_source_generation_match=if_source_generation_match, + if_source_generation_not_match=if_source_generation_not_match, + if_source_metageneration_match=if_source_metageneration_match, + if_source_metageneration_not_match=if_source_metageneration_not_match, + timeout=timeout, + retry=retry, + ) - @create_trace_span(name="Storage.Blob.open") def open( self, mode="r", @@ -4086,51 +4084,54 @@ def open( 'google.cloud.storage.fileio', or an 'io.TextIOWrapper' around one of those classes, depending on the 'mode' argument. """ - if mode == "rb": - if encoding or errors or newline: - raise ValueError( - "encoding, errors and newline arguments are for text mode only" - ) - if ignore_flush: - raise ValueError( - "ignore_flush argument is for non-text write mode only" + with create_trace_span(name="Storage.Blob.open"): + if mode == "rb": + if encoding or errors or newline: + raise ValueError( + "encoding, errors and newline arguments are for text mode only" + ) + if ignore_flush: + raise ValueError( + "ignore_flush argument is for non-text write mode only" + ) + return BlobReader(self, chunk_size=chunk_size, **kwargs) + elif mode == "wb": + if encoding or errors or newline: + raise ValueError( + "encoding, errors and newline arguments are for text mode only" + ) + return BlobWriter( + self, chunk_size=chunk_size, ignore_flush=ignore_flush, **kwargs ) - return BlobReader(self, chunk_size=chunk_size, **kwargs) - elif mode == "wb": - if encoding or errors or newline: - raise ValueError( - "encoding, errors and newline arguments are for text mode only" + elif mode in ("r", "rt"): + if ignore_flush: + raise ValueError( + "ignore_flush argument is for non-text write mode only" + ) + return TextIOWrapper( + BlobReader(self, chunk_size=chunk_size, **kwargs), + encoding=encoding, + errors=errors, + newline=newline, ) - return BlobWriter( - self, chunk_size=chunk_size, ignore_flush=ignore_flush, **kwargs - ) - elif mode in ("r", "rt"): - if ignore_flush: - raise ValueError( - "ignore_flush argument is for non-text write mode only" + elif mode in ("w", "wt"): + if ignore_flush is False: + raise ValueError( + "ignore_flush is required for text mode writing and " + "cannot be set to False" + ) + return TextIOWrapper( + BlobWriter( + self, chunk_size=chunk_size, ignore_flush=True, **kwargs + ), + encoding=encoding, + errors=errors, + newline=newline, ) - return TextIOWrapper( - BlobReader(self, chunk_size=chunk_size, **kwargs), - encoding=encoding, - errors=errors, - newline=newline, - ) - elif mode in ("w", "wt"): - if ignore_flush is False: - raise ValueError( - "ignore_flush is required for text mode writing and " - "cannot be set to False" + else: + raise NotImplementedError( + "Supported modes strings are 'r', 'rb', 'rt', 'w', 'wb', and 'wt' only." ) - return TextIOWrapper( - BlobWriter(self, chunk_size=chunk_size, ignore_flush=True, **kwargs), - encoding=encoding, - errors=errors, - newline=newline, - ) - else: - raise NotImplementedError( - "Supported modes strings are 'r', 'rb', 'rt', 'w', 'wb', and 'wt' only." - ) cache_control = _scalar_property("cacheControl") """HTTP 'Cache-Control' header for this object. diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index 10156c795..fc5733bd0 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -910,7 +910,6 @@ def notification( notification_id=notification_id, ) - @create_trace_span(name="Storage.Bucket.exists") def exists( self, client=None, @@ -958,44 +957,46 @@ def exists( :rtype: bool :returns: True if the bucket exists in Cloud Storage. """ - client = self._require_client(client) - # We only need the status code (200 or not) so we seek to - # minimize the returned payload. - query_params = {"fields": "name"} - - if self.user_project is not None: - query_params["userProject"] = self.user_project + with create_trace_span(name="Storage.Bucket.exists"): + client = self._require_client(client) + # We only need the status code (200 or not) so we seek to + # minimize the returned payload. + query_params = {"fields": "name"} - _add_generation_match_parameters( - query_params, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) + if self.user_project is not None: + query_params["userProject"] = self.user_project - headers = {} - _add_etag_match_headers( - headers, if_etag_match=if_etag_match, if_etag_not_match=if_etag_not_match - ) + _add_generation_match_parameters( + query_params, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) - try: - # We intentionally pass `_target_object=None` since fields=name - # would limit the local properties. - client._get_resource( - self.path, - query_params=query_params, - headers=headers, - timeout=timeout, - retry=retry, - _target_object=None, + headers = {} + _add_etag_match_headers( + headers, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, ) - except NotFound: - # NOTE: This will not fail immediately in a batch. However, when - # Batch.finish() is called, the resulting `NotFound` will be - # raised. - return False - return True - - @create_trace_span(name="Storage.Bucket.create") + + try: + # We intentionally pass `_target_object=None` since fields=name + # would limit the local properties. + client._get_resource( + self.path, + query_params=query_params, + headers=headers, + timeout=timeout, + retry=retry, + _target_object=None, + ) + except NotFound: + # NOTE: This will not fail immediately in a batch. However, when + # Batch.finish() is called, the resulting `NotFound` will be + # raised. + return False + return True + def create( self, client=None, @@ -1057,21 +1058,20 @@ def create( :param retry: (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ + with create_trace_span(name="Storage.Bucket.create"): + client = self._require_client(client) + client.create_bucket( + bucket_or_name=self, + project=project, + user_project=self.user_project, + location=location, + predefined_acl=predefined_acl, + predefined_default_object_acl=predefined_default_object_acl, + enable_object_retention=enable_object_retention, + timeout=timeout, + retry=retry, + ) - client = self._require_client(client) - client.create_bucket( - bucket_or_name=self, - project=project, - user_project=self.user_project, - location=location, - predefined_acl=predefined_acl, - predefined_default_object_acl=predefined_default_object_acl, - enable_object_retention=enable_object_retention, - timeout=timeout, - retry=retry, - ) - - @create_trace_span(name="Storage.Bucket.update") def update( self, client=None, @@ -1108,15 +1108,15 @@ def update( :param retry: (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ - super(Bucket, self).update( - client=client, - timeout=timeout, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - ) + with create_trace_span(name="Storage.Bucket.update"): + super(Bucket, self).update( + client=client, + timeout=timeout, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, + ) - @create_trace_span(name="Storage.Bucket.reload") def reload( self, client=None, @@ -1175,19 +1175,19 @@ def reload( set if ``soft_deleted`` is set to True. See: https://cloud.google.com/storage/docs/soft-delete """ - super(Bucket, self).reload( - client=client, - projection=projection, - timeout=timeout, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - soft_deleted=soft_deleted, - ) + with create_trace_span(name="Storage.Bucket.reload"): + super(Bucket, self).reload( + client=client, + projection=projection, + timeout=timeout, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, + soft_deleted=soft_deleted, + ) - @create_trace_span(name="Storage.Bucket.patch") def patch( self, client=None, @@ -1224,22 +1224,23 @@ def patch( :param retry: (Optional) How to retry the RPC. See: :ref:`configuring_retries` """ - # Special case: For buckets, it is possible that labels are being - # removed; this requires special handling. - if self._label_removals: - self._changes.add("labels") - self._properties.setdefault("labels", {}) - for removed_label in self._label_removals: - self._properties["labels"][removed_label] = None - - # Call the superclass method. - super(Bucket, self).patch( - client=client, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - retry=retry, - ) + with create_trace_span(name="Storage.Bucket.patch"): + # Special case: For buckets, it is possible that labels are being + # removed; this requires special handling. + if self._label_removals: + self._changes.add("labels") + self._properties.setdefault("labels", {}) + for removed_label in self._label_removals: + self._properties["labels"][removed_label] = None + + # Call the superclass method. + super(Bucket, self).patch( + client=client, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + retry=retry, + ) @property def acl(self): @@ -1271,7 +1272,6 @@ def path(self): return self.path_helper(self.name) - @create_trace_span(name="Storage.Bucket.getBlob") def get_blob( self, blob_name, @@ -1360,35 +1360,35 @@ def get_blob( :rtype: :class:`google.cloud.storage.blob.Blob` or None :returns: The blob object if it exists, otherwise None. """ - blob = Blob( - bucket=self, - name=blob_name, - encryption_key=encryption_key, - generation=generation, - **kwargs, - ) - try: - # NOTE: This will not fail immediately in a batch. However, when - # Batch.finish() is called, the resulting `NotFound` will be - # raised. - blob.reload( - client=client, - timeout=timeout, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - soft_deleted=soft_deleted, + with create_trace_span(name="Storage.Bucket.getBlob"): + blob = Blob( + bucket=self, + name=blob_name, + encryption_key=encryption_key, + generation=generation, + **kwargs, ) - except NotFound: - return None - else: - return blob + try: + # NOTE: This will not fail immediately in a batch. However, when + # Batch.finish() is called, the resulting `NotFound` will be + # raised. + blob.reload( + client=client, + timeout=timeout, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, + soft_deleted=soft_deleted, + ) + except NotFound: + return None + else: + return blob - @create_trace_span(name="Storage.Bucket.listBlobs") def list_blobs( self, max_results=None, @@ -1510,28 +1510,28 @@ def list_blobs( :returns: Iterator of all :class:`~google.cloud.storage.blob.Blob` in this bucket matching the arguments. """ - client = self._require_client(client) - return client.list_blobs( - self, - max_results=max_results, - page_token=page_token, - prefix=prefix, - delimiter=delimiter, - start_offset=start_offset, - end_offset=end_offset, - include_trailing_delimiter=include_trailing_delimiter, - versions=versions, - projection=projection, - fields=fields, - page_size=page_size, - timeout=timeout, - retry=retry, - match_glob=match_glob, - include_folders_as_prefixes=include_folders_as_prefixes, - soft_deleted=soft_deleted, - ) + with create_trace_span(name="Storage.Bucket.listBlobs"): + client = self._require_client(client) + return client.list_blobs( + self, + max_results=max_results, + page_token=page_token, + prefix=prefix, + delimiter=delimiter, + start_offset=start_offset, + end_offset=end_offset, + include_trailing_delimiter=include_trailing_delimiter, + versions=versions, + projection=projection, + fields=fields, + page_size=page_size, + timeout=timeout, + retry=retry, + match_glob=match_glob, + include_folders_as_prefixes=include_folders_as_prefixes, + soft_deleted=soft_deleted, + ) - @create_trace_span(name="Storage.Bucket.listNotifications") def list_notifications( self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY ): @@ -1558,18 +1558,18 @@ def list_notifications( :rtype: list of :class:`.BucketNotification` :returns: notification instances """ - client = self._require_client(client) - path = self.path + "/notificationConfigs" - iterator = client._list_resource( - path, - _item_to_notification, - timeout=timeout, - retry=retry, - ) - iterator.bucket = self - return iterator + with create_trace_span(name="Storage.Bucket.listNotifications"): + client = self._require_client(client) + path = self.path + "/notificationConfigs" + iterator = client._list_resource( + path, + _item_to_notification, + timeout=timeout, + retry=retry, + ) + iterator.bucket = self + return iterator - @create_trace_span(name="Storage.Bucket.getNotification") def get_notification( self, notification_id, @@ -1603,11 +1603,11 @@ def get_notification( :rtype: :class:`.BucketNotification` :returns: notification instance. """ - notification = self.notification(notification_id=notification_id) - notification.reload(client=client, timeout=timeout, retry=retry) - return notification + with create_trace_span(name="Storage.Bucket.getNotification"): + notification = self.notification(notification_id=notification_id) + notification.reload(client=client, timeout=timeout, retry=retry) + return notification - @create_trace_span(name="Storage.Bucket.delete") def delete( self, force=False, @@ -1663,58 +1663,58 @@ def delete( :raises: :class:`ValueError` if ``force`` is ``True`` and the bucket contains more than 256 objects / blobs. """ - client = self._require_client(client) - query_params = {} + with create_trace_span(name="Storage.Bucket.delete"): + client = self._require_client(client) + query_params = {} - if self.user_project is not None: - query_params["userProject"] = self.user_project + if self.user_project is not None: + query_params["userProject"] = self.user_project - _add_generation_match_parameters( - query_params, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) - if force: - blobs = list( - self.list_blobs( - max_results=self._MAX_OBJECTS_FOR_ITERATION + 1, + _add_generation_match_parameters( + query_params, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) + if force: + blobs = list( + self.list_blobs( + max_results=self._MAX_OBJECTS_FOR_ITERATION + 1, + client=client, + timeout=timeout, + retry=retry, + versions=True, + ) + ) + if len(blobs) > self._MAX_OBJECTS_FOR_ITERATION: + message = ( + "Refusing to delete bucket with more than " + "%d objects. If you actually want to delete " + "this bucket, please delete the objects " + "yourself before calling Bucket.delete()." + ) % (self._MAX_OBJECTS_FOR_ITERATION,) + raise ValueError(message) + + # Ignore 404 errors on delete. + self.delete_blobs( + blobs, + on_error=lambda blob: None, client=client, timeout=timeout, retry=retry, - versions=True, + preserve_generation=True, ) - ) - if len(blobs) > self._MAX_OBJECTS_FOR_ITERATION: - message = ( - "Refusing to delete bucket with more than " - "%d objects. If you actually want to delete " - "this bucket, please delete the objects " - "yourself before calling Bucket.delete()." - ) % (self._MAX_OBJECTS_FOR_ITERATION,) - raise ValueError(message) - - # Ignore 404 errors on delete. - self.delete_blobs( - blobs, - on_error=lambda blob: None, - client=client, + + # We intentionally pass `_target_object=None` since a DELETE + # request has no response value (whether in a standard request or + # in a batch request). + client._delete_resource( + self.path, + query_params=query_params, timeout=timeout, retry=retry, - preserve_generation=True, + _target_object=None, ) - # We intentionally pass `_target_object=None` since a DELETE - # request has no response value (whether in a standard request or - # in a batch request). - client._delete_resource( - self.path, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=None, - ) - - @create_trace_span(name="Storage.Bucket.deleteBlob") def delete_blob( self, blob_name, @@ -1786,29 +1786,29 @@ def delete_blob( the exception, use :meth:`delete_blobs` by passing a no-op ``on_error`` callback. """ - client = self._require_client(client) - blob = Blob(blob_name, bucket=self, generation=generation) - - query_params = copy.deepcopy(blob._query_params) - _add_generation_match_parameters( - query_params, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) - # We intentionally pass `_target_object=None` since a DELETE - # request has no response value (whether in a standard request or - # in a batch request). - client._delete_resource( - blob.path, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=None, - ) + with create_trace_span(name="Storage.Bucket.deleteBlob"): + client = self._require_client(client) + blob = Blob(blob_name, bucket=self, generation=generation) + + query_params = copy.deepcopy(blob._query_params) + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) + # We intentionally pass `_target_object=None` since a DELETE + # request has no response value (whether in a standard request or + # in a batch request). + client._delete_resource( + blob.path, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=None, + ) - @create_trace_span(name="Storage.Bucket.deleteBlobs") def delete_blobs( self, blobs, @@ -1899,44 +1899,46 @@ def delete_blobs( :raises: :class:`~google.cloud.exceptions.NotFound` (if `on_error` is not passed). """ - _raise_if_len_differs( - len(blobs), - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) - if_generation_match = iter(if_generation_match or []) - if_generation_not_match = iter(if_generation_not_match or []) - if_metageneration_match = iter(if_metageneration_match or []) - if_metageneration_not_match = iter(if_metageneration_not_match or []) + with create_trace_span(name="Storage.Bucket.deleteBlobs"): + _raise_if_len_differs( + len(blobs), + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) + if_generation_match = iter(if_generation_match or []) + if_generation_not_match = iter(if_generation_not_match or []) + if_metageneration_match = iter(if_metageneration_match or []) + if_metageneration_not_match = iter(if_metageneration_not_match or []) - for blob in blobs: - try: - blob_name = blob - generation = None - if not isinstance(blob_name, str): - blob_name = blob.name - generation = blob.generation if preserve_generation else None - - self.delete_blob( - blob_name, - client=client, - generation=generation, - if_generation_match=next(if_generation_match, None), - if_generation_not_match=next(if_generation_not_match, None), - if_metageneration_match=next(if_metageneration_match, None), - if_metageneration_not_match=next(if_metageneration_not_match, None), - timeout=timeout, - retry=retry, - ) - except NotFound: - if on_error is not None: - on_error(blob) - else: - raise + for blob in blobs: + try: + blob_name = blob + generation = None + if not isinstance(blob_name, str): + blob_name = blob.name + generation = blob.generation if preserve_generation else None + + self.delete_blob( + blob_name, + client=client, + generation=generation, + if_generation_match=next(if_generation_match, None), + if_generation_not_match=next(if_generation_not_match, None), + if_metageneration_match=next(if_metageneration_match, None), + if_metageneration_not_match=next( + if_metageneration_not_match, None + ), + timeout=timeout, + retry=retry, + ) + except NotFound: + if on_error is not None: + on_error(blob) + else: + raise - @create_trace_span(name="Storage.Bucket.copyBlob") def copy_blob( self, blob, @@ -2051,48 +2053,48 @@ def copy_blob( :rtype: :class:`google.cloud.storage.blob.Blob` :returns: The new Blob. """ - client = self._require_client(client) - query_params = {} - - if self.user_project is not None: - query_params["userProject"] = self.user_project - - if source_generation is not None: - query_params["sourceGeneration"] = source_generation - - _add_generation_match_parameters( - query_params, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - if_source_generation_match=if_source_generation_match, - if_source_generation_not_match=if_source_generation_not_match, - if_source_metageneration_match=if_source_metageneration_match, - if_source_metageneration_not_match=if_source_metageneration_not_match, - ) + with create_trace_span(name="Storage.Bucket.copyBlob"): + client = self._require_client(client) + query_params = {} - if new_name is None: - new_name = blob.name - - new_blob = Blob(bucket=destination_bucket, name=new_name) - api_path = blob.path + "/copyTo" + new_blob.path - copy_result = client._post_resource( - api_path, - None, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=new_blob, - ) + if self.user_project is not None: + query_params["userProject"] = self.user_project + + if source_generation is not None: + query_params["sourceGeneration"] = source_generation + + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + if_source_generation_match=if_source_generation_match, + if_source_generation_not_match=if_source_generation_not_match, + if_source_metageneration_match=if_source_metageneration_match, + if_source_metageneration_not_match=if_source_metageneration_not_match, + ) + + if new_name is None: + new_name = blob.name + + new_blob = Blob(bucket=destination_bucket, name=new_name) + api_path = blob.path + "/copyTo" + new_blob.path + copy_result = client._post_resource( + api_path, + None, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=new_blob, + ) - if not preserve_acl: - new_blob.acl.save(acl={}, client=client, timeout=timeout) + if not preserve_acl: + new_blob.acl.save(acl={}, client=client, timeout=timeout) - new_blob._set_properties(copy_result) - return new_blob + new_blob._set_properties(copy_result) + return new_blob - @create_trace_span(name="Storage.Bucket.renameBlob") def rename_blob( self, blob, @@ -2205,38 +2207,162 @@ def rename_blob( :rtype: :class:`Blob` :returns: The newly-renamed blob. """ - same_name = blob.name == new_name - - new_blob = self.copy_blob( - blob, - self, - new_name, - client=client, - timeout=timeout, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - if_source_generation_match=if_source_generation_match, - if_source_generation_not_match=if_source_generation_not_match, - if_source_metageneration_match=if_source_metageneration_match, - if_source_metageneration_not_match=if_source_metageneration_not_match, - retry=retry, - ) + with create_trace_span(name="Storage.Bucket.renameBlob"): + same_name = blob.name == new_name - if not same_name: - blob.delete( + new_blob = self.copy_blob( + blob, + self, + new_name, client=client, timeout=timeout, - if_generation_match=if_source_generation_match, - if_generation_not_match=if_source_generation_not_match, - if_metageneration_match=if_source_metageneration_match, - if_metageneration_not_match=if_source_metageneration_not_match, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + if_source_generation_match=if_source_generation_match, + if_source_generation_not_match=if_source_generation_not_match, + if_source_metageneration_match=if_source_metageneration_match, + if_source_metageneration_not_match=if_source_metageneration_not_match, retry=retry, ) - return new_blob - @create_trace_span(name="Storage.Bucket.restore_blob") + if not same_name: + blob.delete( + client=client, + timeout=timeout, + if_generation_match=if_source_generation_match, + if_generation_not_match=if_source_generation_not_match, + if_metageneration_match=if_source_metageneration_match, + if_metageneration_not_match=if_source_metageneration_not_match, + retry=retry, + ) + return new_blob + + def move_blob( + self, + blob, + new_name, + client=None, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, + if_source_generation_match=None, + if_source_generation_not_match=None, + if_source_metageneration_match=None, + if_source_metageneration_not_match=None, + timeout=_DEFAULT_TIMEOUT, + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + ): + """Move a blob to a new name within a single HNS bucket. + + *This feature is currently only supported for HNS (Heirarchical + Namespace) buckets.* + + If :attr:`user_project` is set on the bucket, bills the API request to that project. + + :type blob: :class:`google.cloud.storage.blob.Blob` + :param blob: The blob to be renamed. + + :type new_name: str + :param new_name: The new name for this blob. + + :type client: :class:`~google.cloud.storage.client.Client` or + ``NoneType`` + :param client: (Optional) The client to use. If not passed, falls back + to the ``client`` stored on the current bucket. + + :type if_generation_match: int + :param if_generation_match: + (Optional) See :ref:`using-if-generation-match` + Note that the generation to be matched is that of the + ``destination`` blob. + + :type if_generation_not_match: int + :param if_generation_not_match: + (Optional) See :ref:`using-if-generation-not-match` + Note that the generation to be matched is that of the + ``destination`` blob. + + :type if_metageneration_match: int + :param if_metageneration_match: + (Optional) See :ref:`using-if-metageneration-match` + Note that the metageneration to be matched is that of the + ``destination`` blob. + + :type if_metageneration_not_match: int + :param if_metageneration_not_match: + (Optional) See :ref:`using-if-metageneration-not-match` + Note that the metageneration to be matched is that of the + ``destination`` blob. + + :type if_source_generation_match: int + :param if_source_generation_match: + (Optional) Makes the operation conditional on whether the source + object's generation matches the given value. + + :type if_source_generation_not_match: int + :param if_source_generation_not_match: + (Optional) Makes the operation conditional on whether the source + object's generation does not match the given value. + + :type if_source_metageneration_match: int + :param if_source_metageneration_match: + (Optional) Makes the operation conditional on whether the source + object's current metageneration matches the given value. + + :type if_source_metageneration_not_match: int + :param if_source_metageneration_not_match: + (Optional) Makes the operation conditional on whether the source + object's current metageneration does not match the given value. + + :type timeout: float or tuple + :param timeout: + (Optional) The amount of time, in seconds, to wait + for the server response. See: :ref:`configuring_timeouts` + + :type retry: google.api_core.retry.Retry + :param retry: + (Optional) How to retry the RPC. + See [Configuring Retries](https://cloud.google.com/python/docs/reference/storage/latest/retry_timeout). + + :rtype: :class:`Blob` + :returns: The newly-moved blob. + """ + with create_trace_span(name="Storage.Bucket.moveBlob"): + client = self._require_client(client) + query_params = {} + + if self.user_project is not None: + query_params["userProject"] = self.user_project + + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + if_source_generation_match=if_source_generation_match, + if_source_generation_not_match=if_source_generation_not_match, + if_source_metageneration_match=if_source_metageneration_match, + if_source_metageneration_not_match=if_source_metageneration_not_match, + ) + + new_blob = Blob(bucket=self, name=new_name) + api_path = blob.path + "/moveTo/o/" + new_blob.name + move_result = client._post_resource( + api_path, + None, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=new_blob, + ) + + new_blob._set_properties(move_result) + return new_blob + def restore_blob( self, blob_name, @@ -2310,36 +2436,37 @@ def restore_blob( :rtype: :class:`google.cloud.storage.blob.Blob` :returns: The restored Blob. """ - client = self._require_client(client) - query_params = {} - - if self.user_project is not None: - query_params["userProject"] = self.user_project - if generation is not None: - query_params["generation"] = generation - if copy_source_acl is not None: - query_params["copySourceAcl"] = copy_source_acl - if projection is not None: - query_params["projection"] = projection - - _add_generation_match_parameters( - query_params, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - ) + with create_trace_span(name="Storage.Bucket.restore_blob"): + client = self._require_client(client) + query_params = {} + + if self.user_project is not None: + query_params["userProject"] = self.user_project + if generation is not None: + query_params["generation"] = generation + if copy_source_acl is not None: + query_params["copySourceAcl"] = copy_source_acl + if projection is not None: + query_params["projection"] = projection + + _add_generation_match_parameters( + query_params, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) - blob = Blob(bucket=self, name=blob_name) - api_response = client._post_resource( - f"{blob.path}/restore", - None, - query_params=query_params, - timeout=timeout, - retry=retry, - ) - blob._set_properties(api_response) - return blob + blob = Blob(bucket=self, name=blob_name) + api_response = client._post_resource( + f"{blob.path}/restore", + None, + query_params=query_params, + timeout=timeout, + retry=retry, + ) + blob._set_properties(api_response) + return blob @property def cors(self): @@ -3138,7 +3265,6 @@ def disable_website(self): """ return self.configure_website(None, None) - @create_trace_span(name="Storage.Bucket.getIamPolicy") def get_iam_policy( self, client=None, @@ -3183,25 +3309,25 @@ def get_iam_policy( :returns: the policy instance, based on the resource returned from the ``getIamPolicy`` API request. """ - client = self._require_client(client) - query_params = {} + with create_trace_span(name="Storage.Bucket.getIamPolicy"): + client = self._require_client(client) + query_params = {} - if self.user_project is not None: - query_params["userProject"] = self.user_project + if self.user_project is not None: + query_params["userProject"] = self.user_project - if requested_policy_version is not None: - query_params["optionsRequestedPolicyVersion"] = requested_policy_version + if requested_policy_version is not None: + query_params["optionsRequestedPolicyVersion"] = requested_policy_version - info = client._get_resource( - f"{self.path}/iam", - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=None, - ) - return Policy.from_api_repr(info) + info = client._get_resource( + f"{self.path}/iam", + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=None, + ) + return Policy.from_api_repr(info) - @create_trace_span(name="Storage.Bucket.setIamPolicy") def set_iam_policy( self, policy, @@ -3237,28 +3363,28 @@ def set_iam_policy( :returns: the policy instance, based on the resource returned from the ``setIamPolicy`` API request. """ - client = self._require_client(client) - query_params = {} - - if self.user_project is not None: - query_params["userProject"] = self.user_project - - path = f"{self.path}/iam" - resource = policy.to_api_repr() - resource["resourceId"] = self.path - - info = client._put_resource( - path, - resource, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=None, - ) + with create_trace_span(name="Storage.Bucket.setIamPolicy"): + client = self._require_client(client) + query_params = {} + + if self.user_project is not None: + query_params["userProject"] = self.user_project - return Policy.from_api_repr(info) + path = f"{self.path}/iam" + resource = policy.to_api_repr() + resource["resourceId"] = self.path + + info = client._put_resource( + path, + resource, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=None, + ) + + return Policy.from_api_repr(info) - @create_trace_span(name="Storage.Bucket.testIamPermissions") def test_iam_permissions( self, permissions, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY ): @@ -3290,23 +3416,23 @@ def test_iam_permissions( :returns: the permissions returned by the ``testIamPermissions`` API request. """ - client = self._require_client(client) - query_params = {"permissions": permissions} - - if self.user_project is not None: - query_params["userProject"] = self.user_project - - path = f"{self.path}/iam/testPermissions" - resp = client._get_resource( - path, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=None, - ) - return resp.get("permissions", []) + with create_trace_span(name="Storage.Bucket.testIamPermissions"): + client = self._require_client(client) + query_params = {"permissions": permissions} + + if self.user_project is not None: + query_params["userProject"] = self.user_project + + path = f"{self.path}/iam/testPermissions" + resp = client._get_resource( + path, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=None, + ) + return resp.get("permissions", []) - @create_trace_span(name="Storage.Bucket.makePublic") def make_public( self, recursive=False, @@ -3356,21 +3482,9 @@ def make_public( :meth:`~google.cloud.storage.blob.Blob.make_public` for each blob. """ - self.acl.all().grant_read() - self.acl.save( - client=client, - timeout=timeout, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - ) - - if future: - doa = self.default_object_acl - if not doa.loaded: - doa.reload(client=client, timeout=timeout) - doa.all().grant_read() - doa.save( + with create_trace_span(name="Storage.Bucket.makePublic"): + self.acl.all().grant_read() + self.acl.save( client=client, timeout=timeout, if_metageneration_match=if_metageneration_match, @@ -3378,33 +3492,45 @@ def make_public( retry=retry, ) - if recursive: - blobs = list( - self.list_blobs( - projection="full", - max_results=self._MAX_OBJECTS_FOR_ITERATION + 1, + if future: + doa = self.default_object_acl + if not doa.loaded: + doa.reload(client=client, timeout=timeout) + doa.all().grant_read() + doa.save( client=client, timeout=timeout, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, ) - ) - if len(blobs) > self._MAX_OBJECTS_FOR_ITERATION: - message = ( - "Refusing to make public recursively with more than " - "%d objects. If you actually want to make every object " - "in this bucket public, iterate through the blobs " - "returned by 'Bucket.list_blobs()' and call " - "'make_public' on each one." - ) % (self._MAX_OBJECTS_FOR_ITERATION,) - raise ValueError(message) - for blob in blobs: - blob.acl.all().grant_read() - blob.acl.save( - client=client, - timeout=timeout, + if recursive: + blobs = list( + self.list_blobs( + projection="full", + max_results=self._MAX_OBJECTS_FOR_ITERATION + 1, + client=client, + timeout=timeout, + ) ) + if len(blobs) > self._MAX_OBJECTS_FOR_ITERATION: + message = ( + "Refusing to make public recursively with more than " + "%d objects. If you actually want to make every object " + "in this bucket public, iterate through the blobs " + "returned by 'Bucket.list_blobs()' and call " + "'make_public' on each one." + ) % (self._MAX_OBJECTS_FOR_ITERATION,) + raise ValueError(message) + + for blob in blobs: + blob.acl.all().grant_read() + blob.acl.save( + client=client, + timeout=timeout, + ) - @create_trace_span(name="Storage.Bucket.makePrivate") def make_private( self, recursive=False, @@ -3453,21 +3579,9 @@ def make_private( :meth:`~google.cloud.storage.blob.Blob.make_private` for each blob. """ - self.acl.all().revoke_read() - self.acl.save( - client=client, - timeout=timeout, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - ) - - if future: - doa = self.default_object_acl - if not doa.loaded: - doa.reload(client=client, timeout=timeout) - doa.all().revoke_read() - doa.save( + with create_trace_span(name="Storage.Bucket.makePrivate"): + self.acl.all().revoke_read() + self.acl.save( client=client, timeout=timeout, if_metageneration_match=if_metageneration_match, @@ -3475,28 +3589,41 @@ def make_private( retry=retry, ) - if recursive: - blobs = list( - self.list_blobs( - projection="full", - max_results=self._MAX_OBJECTS_FOR_ITERATION + 1, + if future: + doa = self.default_object_acl + if not doa.loaded: + doa.reload(client=client, timeout=timeout) + doa.all().revoke_read() + doa.save( client=client, timeout=timeout, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, ) - ) - if len(blobs) > self._MAX_OBJECTS_FOR_ITERATION: - message = ( - "Refusing to make private recursively with more than " - "%d objects. If you actually want to make every object " - "in this bucket private, iterate through the blobs " - "returned by 'Bucket.list_blobs()' and call " - "'make_private' on each one." - ) % (self._MAX_OBJECTS_FOR_ITERATION,) - raise ValueError(message) - for blob in blobs: - blob.acl.all().revoke_read() - blob.acl.save(client=client, timeout=timeout) + if recursive: + blobs = list( + self.list_blobs( + projection="full", + max_results=self._MAX_OBJECTS_FOR_ITERATION + 1, + client=client, + timeout=timeout, + ) + ) + if len(blobs) > self._MAX_OBJECTS_FOR_ITERATION: + message = ( + "Refusing to make private recursively with more than " + "%d objects. If you actually want to make every object " + "in this bucket private, iterate through the blobs " + "returned by 'Bucket.list_blobs()' and call " + "'make_private' on each one." + ) % (self._MAX_OBJECTS_FOR_ITERATION,) + raise ValueError(message) + + for blob in blobs: + blob.acl.all().revoke_read() + blob.acl.save(client=client, timeout=timeout) def generate_upload_policy(self, conditions, expiration=None, client=None): """Create a signed upload policy for uploading objects. @@ -3552,7 +3679,6 @@ def generate_upload_policy(self, conditions, expiration=None, client=None): return fields - @create_trace_span(name="Storage.Bucket.lockRetentionPolicy") def lock_retention_policy( self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY ): @@ -3577,34 +3703,39 @@ def lock_retention_policy( if the bucket has no retention policy assigned; if the bucket's retention policy is already locked. """ - if "metageneration" not in self._properties: - raise ValueError("Bucket has no retention policy assigned: try 'reload'?") + with create_trace_span(name="Storage.Bucket.lockRetentionPolicy"): + if "metageneration" not in self._properties: + raise ValueError( + "Bucket has no retention policy assigned: try 'reload'?" + ) - policy = self._properties.get("retentionPolicy") + policy = self._properties.get("retentionPolicy") - if policy is None: - raise ValueError("Bucket has no retention policy assigned: try 'reload'?") + if policy is None: + raise ValueError( + "Bucket has no retention policy assigned: try 'reload'?" + ) - if policy.get("isLocked"): - raise ValueError("Bucket's retention policy is already locked.") + if policy.get("isLocked"): + raise ValueError("Bucket's retention policy is already locked.") - client = self._require_client(client) + client = self._require_client(client) - query_params = {"ifMetagenerationMatch": self.metageneration} + query_params = {"ifMetagenerationMatch": self.metageneration} - if self.user_project is not None: - query_params["userProject"] = self.user_project + if self.user_project is not None: + query_params["userProject"] = self.user_project - path = f"/b/{self.name}/lockRetentionPolicy" - api_response = client._post_resource( - path, - None, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=self, - ) - self._set_properties(api_response) + path = f"/b/{self.name}/lockRetentionPolicy" + api_response = client._post_resource( + path, + None, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=self, + ) + self._set_properties(api_response) def generate_signed_url( self, diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index 57fa7043b..ba94b26fc 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -108,6 +108,12 @@ class Client(ClientWithProject): :param extra_headers: (Optional) Custom headers to be sent with the requests attached to the client. For example, you can add custom audit logging headers. + + :type api_key: string + :param api_key: + (Optional) An API key. Mutually exclusive with any other credentials. + This parameter is an alias for setting `client_options.api_key` and + will supercede any api key set in the `client_options` parameter. """ SCOPE = ( @@ -126,6 +132,8 @@ def __init__( client_options=None, use_auth_w_custom_endpoint=True, extra_headers={}, + *, + api_key=None, ): self._base_connection = None @@ -146,6 +154,17 @@ def __init__( connection_kw_args = {"client_info": client_info} + # api_key should set client_options.api_key. Set it here whether + # client_options was specified as a dict, as a ClientOptions object, or + # None. + if api_key: + if client_options and not isinstance(client_options, dict): + client_options.api_key = api_key + else: + if not client_options: + client_options = {} + client_options["api_key"] = api_key + if client_options: if isinstance(client_options, dict): client_options = google.api_core.client_options.from_dict( @@ -339,7 +358,6 @@ def current_batch(self): """ return self._batch_stack.top - @create_trace_span(name="Storage.Client.getServiceAccountEmail") def get_service_account_email( self, project=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY ): @@ -361,12 +379,13 @@ def get_service_account_email( :rtype: str :returns: service account email address """ - if project is None: - project = self.project + with create_trace_span(name="Storage.Client.getServiceAccountEmail"): + if project is None: + project = self.project - path = f"/projects/{project}/serviceAccount" - api_response = self._get_resource(path, timeout=timeout, retry=retry) - return api_response["email_address"] + path = f"/projects/{project}/serviceAccount" + api_response = self._get_resource(path, timeout=timeout, retry=retry) + return api_response["email_address"] def bucket(self, bucket_name, user_project=None, generation=None): """Factory constructor for bucket object. @@ -831,7 +850,6 @@ def _bucket_arg_to_bucket(self, bucket_or_name, generation=None): bucket = Bucket(self, name=bucket_or_name, generation=generation) return bucket - @create_trace_span(name="Storage.Client.getBucket") def get_bucket( self, bucket_or_name, @@ -903,18 +921,18 @@ def get_bucket( google.cloud.exceptions.NotFound If the bucket is not found. """ - bucket = self._bucket_arg_to_bucket(bucket_or_name, generation=generation) - bucket.reload( - client=self, - timeout=timeout, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - soft_deleted=soft_deleted, - ) - return bucket + with create_trace_span(name="Storage.Client.getBucket"): + bucket = self._bucket_arg_to_bucket(bucket_or_name, generation=generation) + bucket.reload( + client=self, + timeout=timeout, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, + soft_deleted=soft_deleted, + ) + return bucket - @create_trace_span(name="Storage.Client.lookupBucket") def lookup_bucket( self, bucket_name, @@ -951,18 +969,18 @@ def lookup_bucket( :rtype: :class:`google.cloud.storage.bucket.Bucket` or ``NoneType`` :returns: The bucket matching the name provided or None if not found. """ - try: - return self.get_bucket( - bucket_name, - timeout=timeout, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - retry=retry, - ) - except NotFound: - return None + with create_trace_span(name="Storage.Client.lookupBucket"): + try: + return self.get_bucket( + bucket_name, + timeout=timeout, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + retry=retry, + ) + except NotFound: + return None - @create_trace_span(name="Storage.Client.createBucket") def create_bucket( self, bucket_or_name, @@ -1043,70 +1061,72 @@ def create_bucket( google.cloud.exceptions.Conflict If the bucket already exists. """ - bucket = self._bucket_arg_to_bucket(bucket_or_name) - query_params = {} + with create_trace_span(name="Storage.Client.createBucket"): + bucket = self._bucket_arg_to_bucket(bucket_or_name) + query_params = {} - if project is None: - project = self.project - - # Use no project if STORAGE_EMULATOR_HOST is set - if self._is_emulator_set: - if project is None: - project = _get_environ_project() if project is None: - project = "" - - # Only include the project parameter if a project is set. - # If a project is not set, falls back to API validation (BadRequest). - if project is not None: - query_params = {"project": project} - - if requester_pays is not None: - warnings.warn( - "requester_pays arg is deprecated. Use Bucket().requester_pays instead.", - PendingDeprecationWarning, - stacklevel=1, - ) - bucket.requester_pays = requester_pays + project = self.project - if predefined_acl is not None: - predefined_acl = BucketACL.validate_predefined(predefined_acl) - query_params["predefinedAcl"] = predefined_acl + # Use no project if STORAGE_EMULATOR_HOST is set + if self._is_emulator_set: + if project is None: + project = _get_environ_project() + if project is None: + project = "" - if predefined_default_object_acl is not None: - predefined_default_object_acl = DefaultObjectACL.validate_predefined( - predefined_default_object_acl - ) - query_params["predefinedDefaultObjectAcl"] = predefined_default_object_acl + # Only include the project parameter if a project is set. + # If a project is not set, falls back to API validation (BadRequest). + if project is not None: + query_params = {"project": project} - if user_project is not None: - query_params["userProject"] = user_project + if requester_pays is not None: + warnings.warn( + "requester_pays arg is deprecated. Use Bucket().requester_pays instead.", + PendingDeprecationWarning, + stacklevel=1, + ) + bucket.requester_pays = requester_pays - if enable_object_retention: - query_params["enableObjectRetention"] = enable_object_retention + if predefined_acl is not None: + predefined_acl = BucketACL.validate_predefined(predefined_acl) + query_params["predefinedAcl"] = predefined_acl - properties = {key: bucket._properties[key] for key in bucket._changes} - properties["name"] = bucket.name + if predefined_default_object_acl is not None: + predefined_default_object_acl = DefaultObjectACL.validate_predefined( + predefined_default_object_acl + ) + query_params[ + "predefinedDefaultObjectAcl" + ] = predefined_default_object_acl - if location is not None: - properties["location"] = location + if user_project is not None: + query_params["userProject"] = user_project - if data_locations is not None: - properties["customPlacementConfig"] = {"dataLocations": data_locations} + if enable_object_retention: + query_params["enableObjectRetention"] = enable_object_retention - api_response = self._post_resource( - "/b", - properties, - query_params=query_params, - timeout=timeout, - retry=retry, - _target_object=bucket, - ) + properties = {key: bucket._properties[key] for key in bucket._changes} + properties["name"] = bucket.name - bucket._set_properties(api_response) - return bucket + if location is not None: + properties["location"] = location + + if data_locations is not None: + properties["customPlacementConfig"] = {"dataLocations": data_locations} + + api_response = self._post_resource( + "/b", + properties, + query_params=query_params, + timeout=timeout, + retry=retry, + _target_object=bucket, + ) + + bucket._set_properties(api_response) + return bucket - @create_trace_span(name="Storage.Client.downloadBlobToFile") def download_blob_to_file( self, blob_or_uri, @@ -1197,28 +1217,27 @@ def download_blob_to_file( (google.cloud.storage.retry) for information on retry types and how to configure them. """ + with create_trace_span(name="Storage.Client.downloadBlobToFile"): + if not isinstance(blob_or_uri, Blob): + blob_or_uri = Blob.from_uri(blob_or_uri) + + blob_or_uri._prep_and_do_download( + file_obj, + client=self, + start=start, + end=end, + raw_download=raw_download, + if_etag_match=if_etag_match, + if_etag_not_match=if_etag_not_match, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + timeout=timeout, + checksum=checksum, + retry=retry, + ) - if not isinstance(blob_or_uri, Blob): - blob_or_uri = Blob.from_uri(blob_or_uri) - - blob_or_uri._prep_and_do_download( - file_obj, - client=self, - start=start, - end=end, - raw_download=raw_download, - if_etag_match=if_etag_match, - if_etag_not_match=if_etag_not_match, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - timeout=timeout, - checksum=checksum, - retry=retry, - ) - - @create_trace_span(name="Storage.Client.listBlobs") def list_blobs( self, bucket_or_name, @@ -1355,60 +1374,60 @@ def list_blobs( As part of the response, you'll also get back an iterator.prefixes entity that lists object names up to and including the requested delimiter. Duplicate entries are omitted from this list. """ - bucket = self._bucket_arg_to_bucket(bucket_or_name) + with create_trace_span(name="Storage.Client.listBlobs"): + bucket = self._bucket_arg_to_bucket(bucket_or_name) - extra_params = {"projection": projection} + extra_params = {"projection": projection} - if prefix is not None: - extra_params["prefix"] = prefix + if prefix is not None: + extra_params["prefix"] = prefix - if delimiter is not None: - extra_params["delimiter"] = delimiter + if delimiter is not None: + extra_params["delimiter"] = delimiter - if match_glob is not None: - extra_params["matchGlob"] = match_glob + if match_glob is not None: + extra_params["matchGlob"] = match_glob - if start_offset is not None: - extra_params["startOffset"] = start_offset + if start_offset is not None: + extra_params["startOffset"] = start_offset - if end_offset is not None: - extra_params["endOffset"] = end_offset + if end_offset is not None: + extra_params["endOffset"] = end_offset - if include_trailing_delimiter is not None: - extra_params["includeTrailingDelimiter"] = include_trailing_delimiter + if include_trailing_delimiter is not None: + extra_params["includeTrailingDelimiter"] = include_trailing_delimiter - if versions is not None: - extra_params["versions"] = versions + if versions is not None: + extra_params["versions"] = versions - if fields is not None: - extra_params["fields"] = fields + if fields is not None: + extra_params["fields"] = fields - if include_folders_as_prefixes is not None: - extra_params["includeFoldersAsPrefixes"] = include_folders_as_prefixes + if include_folders_as_prefixes is not None: + extra_params["includeFoldersAsPrefixes"] = include_folders_as_prefixes - if soft_deleted is not None: - extra_params["softDeleted"] = soft_deleted + if soft_deleted is not None: + extra_params["softDeleted"] = soft_deleted - if bucket.user_project is not None: - extra_params["userProject"] = bucket.user_project + if bucket.user_project is not None: + extra_params["userProject"] = bucket.user_project - path = bucket.path + "/o" - iterator = self._list_resource( - path, - _item_to_blob, - page_token=page_token, - max_results=max_results, - extra_params=extra_params, - page_start=_blobs_page_start, - page_size=page_size, - timeout=timeout, - retry=retry, - ) - iterator.bucket = bucket - iterator.prefixes = set() - return iterator + path = bucket.path + "/o" + iterator = self._list_resource( + path, + _item_to_blob, + page_token=page_token, + max_results=max_results, + extra_params=extra_params, + page_start=_blobs_page_start, + page_size=page_size, + timeout=timeout, + retry=retry, + ) + iterator.bucket = bucket + iterator.prefixes = set() + return iterator - @create_trace_span(name="Storage.Client.listBuckets") def list_buckets( self, max_results=None, @@ -1486,44 +1505,45 @@ def list_buckets( :returns: Iterator of all :class:`~google.cloud.storage.bucket.Bucket` belonging to this project. """ - extra_params = {} - - if project is None: - project = self.project + with create_trace_span(name="Storage.Client.listBuckets"): + extra_params = {} - # Use no project if STORAGE_EMULATOR_HOST is set - if self._is_emulator_set: if project is None: - project = _get_environ_project() - if project is None: - project = "" + project = self.project - # Only include the project parameter if a project is set. - # If a project is not set, falls back to API validation (BadRequest). - if project is not None: - extra_params = {"project": project} + # Use no project if STORAGE_EMULATOR_HOST is set + if self._is_emulator_set: + if project is None: + project = _get_environ_project() + if project is None: + project = "" - if prefix is not None: - extra_params["prefix"] = prefix + # Only include the project parameter if a project is set. + # If a project is not set, falls back to API validation (BadRequest). + if project is not None: + extra_params = {"project": project} - extra_params["projection"] = projection + if prefix is not None: + extra_params["prefix"] = prefix - if fields is not None: - extra_params["fields"] = fields + extra_params["projection"] = projection - if soft_deleted is not None: - extra_params["softDeleted"] = soft_deleted + if fields is not None: + extra_params["fields"] = fields - return self._list_resource( - "/b", - _item_to_bucket, - page_token=page_token, - max_results=max_results, - extra_params=extra_params, - page_size=page_size, - timeout=timeout, - retry=retry, - ) + if soft_deleted is not None: + extra_params["softDeleted"] = soft_deleted + + return self._list_resource( + "/b", + _item_to_bucket, + page_token=page_token, + max_results=max_results, + extra_params=extra_params, + page_size=page_size, + timeout=timeout, + retry=retry, + ) def restore_bucket( self, @@ -1590,7 +1610,6 @@ def restore_bucket( bucket._set_properties(api_response) return bucket - @create_trace_span(name="Storage.Client.createHmacKey") def create_hmac_key( self, service_account_email, @@ -1634,28 +1653,28 @@ def create_hmac_key( Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str] :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string. """ - if project_id is None: - project_id = self.project + with create_trace_span(name="Storage.Client.createHmacKey"): + if project_id is None: + project_id = self.project - path = f"/projects/{project_id}/hmacKeys" - qs_params = {"serviceAccountEmail": service_account_email} + path = f"/projects/{project_id}/hmacKeys" + qs_params = {"serviceAccountEmail": service_account_email} - if user_project is not None: - qs_params["userProject"] = user_project + if user_project is not None: + qs_params["userProject"] = user_project - api_response = self._post_resource( - path, - None, - query_params=qs_params, - timeout=timeout, - retry=retry, - ) - metadata = HMACKeyMetadata(self) - metadata._properties = api_response["metadata"] - secret = api_response["secret"] - return metadata, secret + api_response = self._post_resource( + path, + None, + query_params=qs_params, + timeout=timeout, + retry=retry, + ) + metadata = HMACKeyMetadata(self) + metadata._properties = api_response["metadata"] + secret = api_response["secret"] + return metadata, secret - @create_trace_span(name="Storage.Client.listHmacKeys") def list_hmac_keys( self, max_results=None, @@ -1701,31 +1720,31 @@ def list_hmac_keys( Tuple[:class:`~google.cloud.storage.hmac_key.HMACKeyMetadata`, str] :returns: metadata for the created key, plus the bytes of the key's secret, which is an 40-character base64-encoded string. """ - if project_id is None: - project_id = self.project + with create_trace_span(name="Storage.Client.listHmacKeys"): + if project_id is None: + project_id = self.project - path = f"/projects/{project_id}/hmacKeys" - extra_params = {} + path = f"/projects/{project_id}/hmacKeys" + extra_params = {} - if service_account_email is not None: - extra_params["serviceAccountEmail"] = service_account_email + if service_account_email is not None: + extra_params["serviceAccountEmail"] = service_account_email - if show_deleted_keys is not None: - extra_params["showDeletedKeys"] = show_deleted_keys + if show_deleted_keys is not None: + extra_params["showDeletedKeys"] = show_deleted_keys - if user_project is not None: - extra_params["userProject"] = user_project + if user_project is not None: + extra_params["userProject"] = user_project - return self._list_resource( - path, - _item_to_hmac_key_metadata, - max_results=max_results, - extra_params=extra_params, - timeout=timeout, - retry=retry, - ) + return self._list_resource( + path, + _item_to_hmac_key_metadata, + max_results=max_results, + extra_params=extra_params, + timeout=timeout, + retry=retry, + ) - @create_trace_span(name="Storage.Client.getHmacKeyMetadata") def get_hmac_key_metadata( self, access_id, project_id=None, user_project=None, timeout=_DEFAULT_TIMEOUT ): @@ -1746,9 +1765,10 @@ def get_hmac_key_metadata( :type user_project: str :param user_project: (Optional) This parameter is currently ignored. """ - metadata = HMACKeyMetadata(self, access_id, project_id, user_project) - metadata.reload(timeout=timeout) # raises NotFound for missing key - return metadata + with create_trace_span(name="Storage.Client.getHmacKeyMetadata"): + metadata = HMACKeyMetadata(self, access_id, project_id, user_project) + metadata.reload(timeout=timeout) # raises NotFound for missing key + return metadata def generate_signed_post_policy_v4( self, diff --git a/google/cloud/storage/hmac_key.py b/google/cloud/storage/hmac_key.py index d37bc071b..547650366 100644 --- a/google/cloud/storage/hmac_key.py +++ b/google/cloud/storage/hmac_key.py @@ -188,7 +188,6 @@ def user_project(self): """ return self._user_project - @create_trace_span(name="Storage.HmacKey.exists") def exists(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): """Determine whether or not the key for this metadata exists. @@ -204,24 +203,24 @@ def exists(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): :rtype: bool :returns: True if the key exists in Cloud Storage. """ - try: - qs_params = {} - - if self.user_project is not None: - qs_params["userProject"] = self.user_project - - self._client._get_resource( - self.path, - query_params=qs_params, - timeout=timeout, - retry=retry, - ) - except NotFound: - return False - else: - return True + with create_trace_span(name="Storage.HmacKey.exists"): + try: + qs_params = {} + + if self.user_project is not None: + qs_params["userProject"] = self.user_project + + self._client._get_resource( + self.path, + query_params=qs_params, + timeout=timeout, + retry=retry, + ) + except NotFound: + return False + else: + return True - @create_trace_span(name="Storage.HmacKey.reload") def reload(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): """Reload properties from Cloud Storage. @@ -237,19 +236,19 @@ def reload(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): :raises :class:`~google.api_core.exceptions.NotFound`: if the key does not exist on the back-end. """ - qs_params = {} + with create_trace_span(name="Storage.HmacKey.reload"): + qs_params = {} - if self.user_project is not None: - qs_params["userProject"] = self.user_project + if self.user_project is not None: + qs_params["userProject"] = self.user_project - self._properties = self._client._get_resource( - self.path, - query_params=qs_params, - timeout=timeout, - retry=retry, - ) + self._properties = self._client._get_resource( + self.path, + query_params=qs_params, + timeout=timeout, + retry=retry, + ) - @create_trace_span(name="Storage.HmacKey.update") def update(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY_IF_ETAG_IN_JSON): """Save writable properties to Cloud Storage. @@ -265,20 +264,20 @@ def update(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY_IF_ETAG_IN_JSON): :raises :class:`~google.api_core.exceptions.NotFound`: if the key does not exist on the back-end. """ - qs_params = {} - if self.user_project is not None: - qs_params["userProject"] = self.user_project - - payload = {"state": self.state} - self._properties = self._client._put_resource( - self.path, - payload, - query_params=qs_params, - timeout=timeout, - retry=retry, - ) - - @create_trace_span(name="Storage.HmacKey.delete") + with create_trace_span(name="Storage.HmacKey.update"): + qs_params = {} + if self.user_project is not None: + qs_params["userProject"] = self.user_project + + payload = {"state": self.state} + self._properties = self._client._put_resource( + self.path, + payload, + query_params=qs_params, + timeout=timeout, + retry=retry, + ) + def delete(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): """Delete the key from Cloud Storage. @@ -294,13 +293,14 @@ def delete(self, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): :raises :class:`~google.api_core.exceptions.NotFound`: if the key does not exist on the back-end. """ - qs_params = {} - if self.user_project is not None: - qs_params["userProject"] = self.user_project - - self._client._delete_resource( - self.path, - query_params=qs_params, - timeout=timeout, - retry=retry, - ) + with create_trace_span(name="Storage.HmacKey.delete"): + qs_params = {} + if self.user_project is not None: + qs_params["userProject"] = self.user_project + + self._client._delete_resource( + self.path, + query_params=qs_params, + timeout=timeout, + retry=retry, + ) diff --git a/google/cloud/storage/notification.py b/google/cloud/storage/notification.py index d9d49fc4b..d13b80fc4 100644 --- a/google/cloud/storage/notification.py +++ b/google/cloud/storage/notification.py @@ -231,7 +231,6 @@ def _set_properties(self, response): self._properties.clear() self._properties.update(response) - @create_trace_span(name="Storage.BucketNotification.create") def create(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=None): """API wrapper: create the notification. @@ -255,36 +254,36 @@ def create(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=None): :raises ValueError: if the notification already exists. """ - if self.notification_id is not None: - raise ValueError( - f"notification_id already set to {self.notification_id}; must be None to create a Notification." - ) - - client = self._require_client(client) - - query_params = {} - if self.bucket.user_project is not None: - query_params["userProject"] = self.bucket.user_project - - path = f"/b/{self.bucket.name}/notificationConfigs" - properties = self._properties.copy() - - if self.topic_name is None: - properties["topic"] = _TOPIC_REF_FMT.format(self.topic_project, "") - else: - properties["topic"] = _TOPIC_REF_FMT.format( - self.topic_project, self.topic_name + with create_trace_span(name="Storage.BucketNotification.create"): + if self.notification_id is not None: + raise ValueError( + f"notification_id already set to {self.notification_id}; must be None to create a Notification." + ) + + client = self._require_client(client) + + query_params = {} + if self.bucket.user_project is not None: + query_params["userProject"] = self.bucket.user_project + + path = f"/b/{self.bucket.name}/notificationConfigs" + properties = self._properties.copy() + + if self.topic_name is None: + properties["topic"] = _TOPIC_REF_FMT.format(self.topic_project, "") + else: + properties["topic"] = _TOPIC_REF_FMT.format( + self.topic_project, self.topic_name + ) + + self._properties = client._post_resource( + path, + properties, + query_params=query_params, + timeout=timeout, + retry=retry, ) - self._properties = client._post_resource( - path, - properties, - query_params=query_params, - timeout=timeout, - retry=retry, - ) - - @create_trace_span(name="Storage.BucketNotification.exists") def exists(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): """Test whether this notification exists. @@ -311,28 +310,30 @@ def exists(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): :returns: True, if the notification exists, else False. :raises ValueError: if the notification has no ID. """ - if self.notification_id is None: - raise ValueError("Notification ID not set: set an explicit notification_id") - - client = self._require_client(client) - - query_params = {} - if self.bucket.user_project is not None: - query_params["userProject"] = self.bucket.user_project + with create_trace_span(name="Storage.BucketNotification.exists"): + if self.notification_id is None: + raise ValueError( + "Notification ID not set: set an explicit notification_id" + ) + + client = self._require_client(client) + + query_params = {} + if self.bucket.user_project is not None: + query_params["userProject"] = self.bucket.user_project + + try: + client._get_resource( + self.path, + query_params=query_params, + timeout=timeout, + retry=retry, + ) + except NotFound: + return False + else: + return True - try: - client._get_resource( - self.path, - query_params=query_params, - timeout=timeout, - retry=retry, - ) - except NotFound: - return False - else: - return True - - @create_trace_span(name="Storage.BucketNotification.reload") def reload(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): """Update this notification from the server configuration. @@ -358,24 +359,26 @@ def reload(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): :raises ValueError: if the notification has no ID. """ - if self.notification_id is None: - raise ValueError("Notification ID not set: set an explicit notification_id") + with create_trace_span(name="Storage.BucketNotification.reload"): + if self.notification_id is None: + raise ValueError( + "Notification ID not set: set an explicit notification_id" + ) - client = self._require_client(client) + client = self._require_client(client) - query_params = {} - if self.bucket.user_project is not None: - query_params["userProject"] = self.bucket.user_project + query_params = {} + if self.bucket.user_project is not None: + query_params["userProject"] = self.bucket.user_project - response = client._get_resource( - self.path, - query_params=query_params, - timeout=timeout, - retry=retry, - ) - self._set_properties(response) + response = client._get_resource( + self.path, + query_params=query_params, + timeout=timeout, + retry=retry, + ) + self._set_properties(response) - @create_trace_span(name="Storage.BucketNotification.delete") def delete(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): """Delete this notification. @@ -402,21 +405,24 @@ def delete(self, client=None, timeout=_DEFAULT_TIMEOUT, retry=DEFAULT_RETRY): if the notification does not exist. :raises ValueError: if the notification has no ID. """ - if self.notification_id is None: - raise ValueError("Notification ID not set: set an explicit notification_id") + with create_trace_span(name="Storage.BucketNotification.delete"): + if self.notification_id is None: + raise ValueError( + "Notification ID not set: set an explicit notification_id" + ) - client = self._require_client(client) + client = self._require_client(client) - query_params = {} - if self.bucket.user_project is not None: - query_params["userProject"] = self.bucket.user_project + query_params = {} + if self.bucket.user_project is not None: + query_params["userProject"] = self.bucket.user_project - client._delete_resource( - self.path, - query_params=query_params, - timeout=timeout, - retry=retry, - ) + client._delete_resource( + self.path, + query_params=query_params, + timeout=timeout, + retry=retry, + ) def _parse_topic_path(topic_path): diff --git a/google/cloud/storage/version.py b/google/cloud/storage/version.py index d6f7def8c..6ce498ba5 100644 --- a/google/cloud/storage/version.py +++ b/google/cloud/storage/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "3.0.0" +__version__ = "3.1.0" diff --git a/setup.py b/setup.py index 84eedd4f2..31c360f40 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ dependencies = [ "google-auth >= 2.26.1, < 3.0dev", "google-api-core >= 2.15.0, <3.0.0dev", - "google-cloud-core >= 2.3.0, < 3.0dev", + "google-cloud-core >= 2.4.2, < 3.0dev", # The dependency "google-resumable-media" is no longer used. However, the # dependency is still included here to accommodate users who may be # importing exception classes from the google-resumable-media without diff --git a/tests/system/test_bucket.py b/tests/system/test_bucket.py index 270a77ad1..f06de8e8c 100644 --- a/tests/system/test_bucket.py +++ b/tests/system/test_bucket.py @@ -433,6 +433,40 @@ def test_bucket_copy_blob_w_metageneration_match( assert new_blob.download_as_bytes() == payload +def test_bucket_move_blob_hns( + storage_client, + buckets_to_delete, + blobs_to_delete, +): + payload = b"move_blob_test" + + # Feature currently only works on HNS buckets, so create one here + bucket_name = _helpers.unique_name("move-blob-hns-enabled") + bucket_obj = storage_client.bucket(bucket_name) + bucket_obj.hierarchical_namespace_enabled = True + bucket_obj.iam_configuration.uniform_bucket_level_access_enabled = True + created = _helpers.retry_429_503(storage_client.create_bucket)(bucket_obj) + buckets_to_delete.append(created) + assert created.hierarchical_namespace_enabled is True + + source = created.blob("source") + source_gen = source.generation + source.upload_from_string(payload) + blobs_to_delete.append(source) + + dest = created.move_blob( + source, + "dest", + if_source_generation_match=source.generation, + if_source_metageneration_match=source.metageneration, + ) + blobs_to_delete.append(dest) + + assert dest.download_as_bytes() == payload + assert dest.generation is not None + assert source_gen != dest.generation + + def test_bucket_get_blob_with_user_project( storage_client, buckets_to_delete, diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index 7129232a0..ac9a5ede6 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -2289,6 +2289,69 @@ def test_copy_blob_w_name_and_user_project(self): _target_object=new_blob, ) + def test_move_blob_w_no_retry_timeout_and_generation_match(self): + source_name = "source" + blob_name = "blob-name" + new_name = "new_name" + api_response = {} + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response + source = self._make_one(client=client, name=source_name) + blob = self._make_blob(source_name, blob_name) + + new_blob = source.move_blob( + blob, new_name, if_generation_match=0, retry=None, timeout=30 + ) + + self.assertIs(new_blob.bucket, source) + self.assertEqual(new_blob.name, new_name) + + expected_path = "/b/{}/o/{}/moveTo/o/{}".format( + source_name, blob_name, new_name + ) + expected_data = None + expected_query_params = {"ifGenerationMatch": 0} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=30, + retry=None, + _target_object=new_blob, + ) + + def test_move_blob_w_user_project(self): + source_name = "source" + blob_name = "blob-name" + new_name = "new_name" + user_project = "user-project-123" + api_response = {} + client = mock.Mock(spec=["_post_resource"]) + client._post_resource.return_value = api_response + source = self._make_one( + client=client, name=source_name, user_project=user_project + ) + blob = self._make_blob(source_name, blob_name) + + new_blob = source.move_blob(blob, new_name) + + self.assertIs(new_blob.bucket, source) + self.assertEqual(new_blob.name, new_name) + + expected_path = "/b/{}/o/{}/moveTo/o/{}".format( + source_name, blob_name, new_name + ) + expected_data = None + expected_query_params = {"userProject": user_project} + client._post_resource.assert_called_once_with( + expected_path, + expected_data, + query_params=expected_query_params, + timeout=self._get_default_timeout(), + retry=DEFAULT_RETRY_IF_GENERATION_SPECIFIED, + _target_object=new_blob, + ) + def _rename_blob_helper(self, explicit_client=False, same_name=False, **kw): bucket_name = "BUCKET_NAME" blob_name = "blob-name" diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 0bef1ea91..b671cc092 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -191,6 +191,53 @@ def test_ctor_w_client_options_object(self): self.assertEqual(client._connection.API_BASE_URL, api_endpoint) self.assertEqual(client.api_endpoint, api_endpoint) + def test_ctor_w_api_key(self): + from google.auth.api_key import Credentials + + PROJECT = "PROJECT" + api_key = "my_api_key" + + client = self._make_one(project=PROJECT, api_key=api_key) + + self.assertEqual( + client._connection.API_BASE_URL, client._connection.DEFAULT_API_ENDPOINT + ) + self.assertIsInstance(client._credentials, Credentials) + self.assertEqual(client._credentials.token, api_key) + + def test_ctor_w_api_key_and_client_options(self): + from google.auth.api_key import Credentials + from google.api_core.client_options import ClientOptions + + PROJECT = "PROJECT" + api_key = "my_api_key" + api_endpoint = "https://www.foo-googleapis.com" + client_options = ClientOptions(api_endpoint=api_endpoint) + + client = self._make_one( + project=PROJECT, client_options=client_options, api_key=api_key + ) + + self.assertEqual(client._connection.API_BASE_URL, api_endpoint) + self.assertIsInstance(client._credentials, Credentials) + self.assertEqual(client._credentials.token, api_key) + + def test_ctor_w_api_key_and_client_dict(self): + from google.auth.api_key import Credentials + + PROJECT = "PROJECT" + api_key = "my_api_key" + api_endpoint = "https://www.foo-googleapis.com" + client_options = {"api_endpoint": api_endpoint} + + client = self._make_one( + project=PROJECT, client_options=client_options, api_key=api_key + ) + + self.assertEqual(client._connection.API_BASE_URL, api_endpoint) + self.assertIsInstance(client._credentials, Credentials) + self.assertEqual(client._credentials.token, api_key) + def test_ctor_w_universe_domain_and_matched_credentials(self): PROJECT = "PROJECT" universe_domain = "example.com"