Skip to content

Commit f669982

Browse files
tritoneBrennaEpp
tritone
and
BrennaEpp
authored
feat(storage): add experimental ZB API option (#12214)
This option causes bidi reads to be enabled and writes to use appendable semantics. This should make it easier for ZB users and to run integration tests against ZB. Co-authored-by: Brenna N Epp <[email protected]>
1 parent 2671c4f commit f669982

File tree

8 files changed

+58
-5
lines changed

8 files changed

+58
-5
lines changed

‎storage/experimental/experimental.go

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,11 +77,25 @@ type ReadStallTimeoutConfig struct {
7777
// WithGRPCBidiReads provides an [option.ClientOption] that may be passed to
7878
// [cloud.google.com/go/storage.NewGRPCClient].
7979
// It enables the client to use bi-directional gRPC APIs for downloads rather than the
80-
// server streaming API. In particular, it allows users to use the [storage.MultiRangeDownloader]
80+
// server streaming API. In particular, it allows users to use the
81+
// [cloud.google.com/go/storage.MultiRangeDownloader]
8182
// surface, which requires bi-directional streaming.
8283
//
8384
// The bi-directional API is in private preview; please contact your account manager if
8485
// interested.
8586
func WithGRPCBidiReads() option.ClientOption {
8687
return internal.WithGRPCBidiReads.(func() option.ClientOption)()
8788
}
89+
90+
// WithZonalBucketAPIs provides an [option.ClientOption] that may be passed to
91+
// [cloud.google.com/go/storage.NewGRPCClient].
92+
// It enables the client to use bi-directional gRPC APIs for downloads rather than the
93+
// server streaming API (same as [WithGRPCBidiReads]) as well as appendable
94+
// object semantics for uploads. By setting this option, both upload and download
95+
// paths will use zonal bucket compatible APIs by default.
96+
//
97+
// Zonal buckets and rapid storage is in private preview; please contact your
98+
// account manager if interested.
99+
func WithZonalBucketAPIs() option.ClientOption {
100+
return internal.WithZonalBucketAPIs.(func() option.ClientOption)()
101+
}

‎storage/grpc_client.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ func enableClientMetrics(ctx context.Context, s *settings, config storageConfig)
132132

133133
// newGRPCStorageClient initializes a new storageClient that uses the gRPC
134134
// Storage API.
135-
func newGRPCStorageClient(ctx context.Context, opts ...storageOption) (storageClient, error) {
135+
func newGRPCStorageClient(ctx context.Context, opts ...storageOption) (*grpcStorageClient, error) {
136136
s := initSettings(opts...)
137137
s.clientOption = append(defaultGRPCOptions(), s.clientOption...)
138138
// Disable all gax-level retries in favor of retry logic in the veneer client.

‎storage/internal/experimental.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,9 @@ var (
3333
// WithGRPCBidiReads is a function which is implemented by the storage package.
3434
// It sets the gRPC client to use the BidiReadObject API for downloads.
3535
WithGRPCBidiReads any // func() option.ClientOption
36+
37+
// WithZonalBucketAPIs is a function which is implemented by the storage package.
38+
// It sets the gRPC client to use the BidiReadObject API for downloads and
39+
// appendable object semantics by default for uploads.
40+
WithZonalBucketAPIs any // func() option.ClientOption
3641
)

‎storage/option.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ func init() {
4141
storageinternal.WithMetricInterval = withMetricInterval
4242
storageinternal.WithReadStallTimeout = withReadStallTimeout
4343
storageinternal.WithGRPCBidiReads = withGRPCBidiReads
44+
storageinternal.WithZonalBucketAPIs = withZonalBucketAPIs
4445
}
4546

4647
// getDynamicReadReqIncreaseRateFromEnv returns the value set in the env variable.
@@ -83,6 +84,7 @@ type storageConfig struct {
8384
manualReader *metric.ManualReader
8485
readStallTimeoutConfig *experimental.ReadStallTimeoutConfig
8586
grpcBidiReads bool
87+
grpcAppendableUploads bool
8688
}
8789

8890
// newStorageConfig generates a new storageConfig with all the given
@@ -254,3 +256,17 @@ type withGRPCBidiReadsConfig struct {
254256
func (w *withGRPCBidiReadsConfig) ApplyStorageOpt(config *storageConfig) {
255257
config.grpcBidiReads = true
256258
}
259+
260+
func withZonalBucketAPIs() option.ClientOption {
261+
return &withZonalBucketAPIsConfig{}
262+
}
263+
264+
type withZonalBucketAPIsConfig struct {
265+
internaloption.EmbeddableAdapter
266+
}
267+
268+
func (w *withZonalBucketAPIsConfig) ApplyStorageOpt(config *storageConfig) {
269+
// Use both appendable upload semantics and bidi reads.
270+
config.grpcAppendableUploads = true
271+
config.grpcBidiReads = true
272+
}

‎storage/option_test.go

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,14 @@ func TestApplyStorageOpt(t *testing.T) {
145145
grpcBidiReads: true,
146146
},
147147
},
148+
{
149+
desc: "use gRPC zonal bucket APIs",
150+
opts: []option.ClientOption{withZonalBucketAPIs()},
151+
want: storageConfig{
152+
grpcBidiReads: true,
153+
grpcAppendableUploads: true,
154+
},
155+
},
148156
} {
149157
t.Run(test.desc, func(t *testing.T) {
150158
var got storageConfig

‎storage/reader.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,8 @@ func (o *ObjectHandle) NewRangeReader(ctx context.Context, offset, length int64)
162162
//
163163
// This uses the gRPC-specific bi-directional read API, which is in private
164164
// preview; please contact your account manager if interested. The option
165-
// [experimental.WithGRPCBidiReads] must be selected in order to use this API.
165+
// [experimental.WithGRPCBidiReads] or [experimental.WithZonalBucketAPIs]
166+
// must be selected in order to use this API.
166167
func (o *ObjectHandle) NewMultiRangeDownloader(ctx context.Context) (mrd *MultiRangeDownloader, err error) {
167168
// This span covers the life of the reader. It is closed via the context
168169
// in Reader.Close.

‎storage/storage.go

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,9 @@ type Client struct {
126126

127127
// tc is the transport-agnostic client implemented with either gRPC or HTTP.
128128
tc storageClient
129+
130+
// Option to use gRRPC appendable upload API was set.
131+
grpcAppendableUploads bool
129132
}
130133

131134
// NewClient creates a new Google Cloud Storage client using the HTTP transport.
@@ -238,8 +241,10 @@ func NewGRPCClient(ctx context.Context, opts ...option.ClientOption) (*Client, e
238241
if err != nil {
239242
return nil, err
240243
}
241-
242-
return &Client{tc: tc}, nil
244+
return &Client{
245+
tc: tc,
246+
grpcAppendableUploads: tc.config.grpcAppendableUploads,
247+
}, nil
243248
}
244249

245250
// CheckDirectConnectivitySupported checks if gRPC direct connectivity
@@ -1240,6 +1245,7 @@ func (o *ObjectHandle) NewWriter(ctx context.Context) *Writer {
12401245
donec: make(chan struct{}),
12411246
ObjectAttrs: ObjectAttrs{Name: o.object},
12421247
ChunkSize: googleapi.DefaultUploadChunkSize,
1248+
Append: o.c.grpcAppendableUploads,
12431249
}
12441250
}
12451251

‎storage/writer.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,9 @@ type Writer struct {
111111
// when Writer.Close() is called; otherwise, the object is left unfinalized
112112
// and can be appended to later.
113113
//
114+
// Defaults to false unless the experiemental WithZonalBucketAPIs option was
115+
// set.
116+
//
114117
// Append is only supported for gRPC. This feature is in preview and is not
115118
// yet available for general use.
116119
Append bool

0 commit comments

Comments
 (0)