googleapi: remove size sniffing, send chunked multipart/mime request …

…bodies In the past, the Google API server required Content-Lengths on requests and didn't accept chunked uploads. That appears to be fixed now, so remove a bunch of complexity in trying to calculate the size of request bodies, the worst of which involved slurping the whole content into memory (which might fail and/or use all available memory). Change-Id: I8995204d604baf08237593b7a077f4dbfd5b7b11 Reviewed-on: https://code-review.googlesource.com/1870 Reviewed-by: Glenn Lewis <[email protected]>
waleed566 · Jan 24, 2015 · 8a09877 · 8a09877
1 parent 3f7dcd0
commit 8a09877
Show file tree

Hide file tree

Showing 146 changed files with 10,462 additions and 731 deletions.
diff --git a/adexchangebuyer/v1.2/adexchangebuyer-gen.go b/adexchangebuyer/v1.2/adexchangebuyer-gen.go
@@ -14,6 +14,7 @@ import (
  "encoding/json"
  "errors"
  "fmt"
+ "golang.org/x/net/context"
  "google.golang.org/api/googleapi"
  "io"
  "net/http"
@@ -33,6 +34,7 @@ var _ = url.Parse
 var _ = googleapi.Version
 var _ = errors.New
 var _ = strings.Replace
+var _ = context.Background
 
 const apiId = "adexchangebuyer:v1.2"
 const apiName = "adexchangebuyer"

diff --git a/adexchangebuyer/v1.3/adexchangebuyer-gen.go b/adexchangebuyer/v1.3/adexchangebuyer-gen.go
@@ -14,6 +14,7 @@ import (
  "encoding/json"
  "errors"
  "fmt"
+ "golang.org/x/net/context"
  "google.golang.org/api/googleapi"
  "io"
  "net/http"
@@ -33,6 +34,7 @@ var _ = url.Parse
 var _ = googleapi.Version
 var _ = errors.New
 var _ = strings.Replace
+var _ = context.Background
 
 const apiId = "adexchangebuyer:v1.3"
 const apiName = "adexchangebuyer"

diff --git a/adexchangeseller/v1.1/adexchangeseller-api.json b/adexchangeseller/v1.1/adexchangeseller-api.json
@@ -1,12 +1,12 @@
 {
  "kind": "discovery#restDescription",
- "etag": "\"ye6orv2F-1npMW3u9suM3a7C5Bo/fcWyClHZWfYoiPcBzxEos_wQ_94\"",
+ "etag": "\"ye6orv2F-1npMW3u9suM3a7C5Bo/Ob-vIbU0CMw39spLQLZEQavYp3Q\"",
  "discoveryVersion": "v1",
  "id": "adexchangeseller:v1.1",
  "name": "adexchangeseller",
  "canonicalName": "Ad Exchange Seller",
  "version": "v1.1",
- "revision": "20141222",
+ "revision": "20141112",
  "title": "Ad Exchange Seller API",
  "description": "Gives Ad Exchange seller users access to their inventory and the ability to generate reports",
  "ownerDomain": "google.com",

diff --git a/adexchangeseller/v1.1/adexchangeseller-gen.go b/adexchangeseller/v1.1/adexchangeseller-gen.go
@@ -14,6 +14,7 @@ import (
  "encoding/json"
  "errors"
  "fmt"
+ "golang.org/x/net/context"
  "google.golang.org/api/googleapi"
  "io"
  "net/http"
@@ -33,6 +34,7 @@ var _ = url.Parse
 var _ = googleapi.Version
 var _ = errors.New
 var _ = strings.Replace
+var _ = context.Background
 
 const apiId = "adexchangeseller:v1.1"
 const apiName = "adexchangeseller"

diff --git a/adexchangeseller/v1/adexchangeseller-api.json b/adexchangeseller/v1/adexchangeseller-api.json
@@ -1,12 +1,12 @@
 {
  "kind": "discovery#restDescription",
- "etag": "\"ye6orv2F-1npMW3u9suM3a7C5Bo/rndP_70v9IVpLK84zn_P-mVlnXI\"",
+ "etag": "\"ye6orv2F-1npMW3u9suM3a7C5Bo/kgKc4O1bceVftsMmf3p1OXF57gg\"",
  "discoveryVersion": "v1",
  "id": "adexchangeseller:v1",
  "name": "adexchangeseller",
  "canonicalName": "Ad Exchange Seller",
  "version": "v1",
- "revision": "20141222",
+ "revision": "20141112",
  "title": "Ad Exchange Seller API",
  "description": "Gives Ad Exchange seller users access to their inventory and the ability to generate reports",
  "ownerDomain": "google.com",

diff --git a/adexchangeseller/v1/adexchangeseller-gen.go b/adexchangeseller/v1/adexchangeseller-gen.go
@@ -14,6 +14,7 @@ import (
  "encoding/json"
  "errors"
  "fmt"
+ "golang.org/x/net/context"
  "google.golang.org/api/googleapi"
  "io"
  "net/http"
@@ -33,6 +34,7 @@ var _ = url.Parse
 var _ = googleapi.Version
 var _ = errors.New
 var _ = strings.Replace
+var _ = context.Background
 
 const apiId = "adexchangeseller:v1"
 const apiName = "adexchangeseller"

diff --git a/adexchangeseller/v2.0/adexchangeseller-api.json b/adexchangeseller/v2.0/adexchangeseller-api.json
@@ -1,12 +1,12 @@
 {
  "kind": "discovery#restDescription",
- "etag": "\"ye6orv2F-1npMW3u9suM3a7C5Bo/L4L12_fshxMlVyFutADZBQXnkNs\"",
+ "etag": "\"ye6orv2F-1npMW3u9suM3a7C5Bo/6aC1_omptpkys969gj83ZtJAvZ8\"",
  "discoveryVersion": "v1",
  "id": "adexchangeseller:v2.0",
  "name": "adexchangeseller",
  "canonicalName": "Ad Exchange Seller",
  "version": "v2.0",
- "revision": "20141222",
+ "revision": "20141112",
  "title": "Ad Exchange Seller API",
  "description": "Gives Ad Exchange seller users access to their inventory and the ability to generate reports",
  "ownerDomain": "google.com",

diff --git a/adexchangeseller/v2.0/adexchangeseller-gen.go b/adexchangeseller/v2.0/adexchangeseller-gen.go
@@ -14,6 +14,7 @@ import (
  "encoding/json"
  "errors"
  "fmt"
+ "golang.org/x/net/context"
  "google.golang.org/api/googleapi"
  "io"
  "net/http"
@@ -33,6 +34,7 @@ var _ = url.Parse
 var _ = googleapi.Version
 var _ = errors.New
 var _ = strings.Replace
+var _ = context.Background
 
 const apiId = "adexchangeseller:v2.0"
 const apiName = "adexchangeseller"

diff --git a/admin/directory_v1/admin-gen.go b/admin/directory_v1/admin-gen.go
@@ -14,6 +14,7 @@ import (
  "encoding/json"
  "errors"
  "fmt"
+ "golang.org/x/net/context"
  "google.golang.org/api/googleapi"
  "io"
  "net/http"
@@ -33,6 +34,7 @@ var _ = url.Parse
 var _ = googleapi.Version
 var _ = errors.New
 var _ = strings.Replace
+var _ = context.Background
 
 const apiId = "admin:directory_v1"
 const apiName = "admin"

diff --git a/admin/email_migration_v2/admin-gen.go b/admin/email_migration_v2/admin-gen.go
@@ -14,6 +14,7 @@ import (
  "encoding/json"
  "errors"
  "fmt"
+ "golang.org/x/net/context"
  "google.golang.org/api/googleapi"
  "io"
  "net/http"
@@ -33,6 +34,7 @@ var _ = url.Parse
 var _ = googleapi.Version
 var _ = errors.New
 var _ = strings.Replace
+var _ = context.Background
 
 const apiId = "admin:email_migration_v2"
 const apiName = "admin"
@@ -102,11 +104,15 @@ type MailItem struct {
 // method id "emailMigration.mail.insert":
 
 type MailInsertCall struct {
- s *Service
- userKey string
- mailitem *MailItem
- opt_ map[string]interface{}
- media_ io.Reader
+ s *Service
+ userKey string
+ mailitem *MailItem
+ opt_ map[string]interface{}
+ media_ io.Reader
+ resumable_ googleapi.SizeReaderAt
+ mediaType_ string
+ ctx_ context.Context
+ protocol_ string
 }
 
 // Insert: Insert Mail into Google's Gmail backends
@@ -116,8 +122,32 @@ func (r *MailService) Insert(userKey string, mailitem *MailItem) *MailInsertCall
  c.mailitem = mailitem
  return c
 }
+
+// Media specifies the media to upload in a single chunk.
+// At most one of Media and ResumableMedia may be set.
 func (c *MailInsertCall) Media(r io.Reader) *MailInsertCall {
  c.media_ = r
+ c.protocol_ = "multipart"
+ return c
+}
+
+// ResumableMedia specifies the media to upload in chunks and can be cancelled with ctx.
+// At most one of Media and ResumableMedia may be set.
+// mediaType identifies the MIME media type of the upload, such as "image/png".
+// If mediaType is "", it will be auto-detected.
+func (c *MailInsertCall) ResumableMedia(ctx context.Context, r io.ReaderAt, size int64, mediaType string) *MailInsertCall {
+ c.ctx_ = ctx
+ c.resumable_ = io.NewSectionReader(r, 0, size)
+ c.mediaType_ = mediaType
+ c.protocol_ = "resumable"
+ return c
+}
+
+// ProgressUpdater provides a callback function that will be called after every chunk.
+// It should be a low-latency function in order to not slow down the upload operation.
+// This should only be called when using ResumableMedia (as opposed to Media).
+func (c *MailInsertCall) ProgressUpdater(pu googleapi.ProgressUpdater) *MailInsertCall {
+ c.opt_["progressUpdater"] = pu
  return c
 }
 
@@ -142,20 +172,42 @@ func (c *MailInsertCall) Do() error {
  params.Set("fields", fmt.Sprintf("%v", v))
  }
  urls := googleapi.ResolveRelative(c.s.BasePath, "{userKey}/mail")
- if c.media_ != nil {
+ var progressUpdater_ googleapi.ProgressUpdater
+ if v, ok := c.opt_["progressUpdater"]; ok {
+ if pu, ok := v.(googleapi.ProgressUpdater); ok {
+ progressUpdater_ = pu
+ }
+ }
+ if c.media_ != nil || c.resumable_ != nil {
  urls = strings.Replace(urls, "https://www.googleapis.com/", "https://www.googleapis.com/upload/", 1)
- params.Set("uploadType", "multipart")
+ params.Set("uploadType", c.protocol_)
  }
  urls += "?" + params.Encode()
- contentLength_, hasMedia_ := googleapi.ConditionallyIncludeMedia(c.media_, &body, &ctype)
+ var hasMedia_ bool
+ if c.protocol_ != "resumable" {
+ var cancel func()
+ cancel, hasMedia_ = googleapi.ConditionallyIncludeMedia(c.media_, &body, &ctype)
+ if cancel != nil {
+ defer cancel()
+ }
+ }
  req, _ := http.NewRequest("POST", urls, body)
  googleapi.Expand(req.URL, map[string]string{
  "userKey": c.userKey,
  })
- if hasMedia_ {
- req.ContentLength = contentLength_
+ if c.protocol_ == "resumable" {
+ req.ContentLength = 0
+ if c.mediaType_ == "" {
+ c.mediaType_ = googleapi.DetectMediaType(c.resumable_)
+ }
+ req.Header.Set("X-Upload-Content-Type", c.mediaType_)
+ req.Body = nil
+ if params.Get("name") == "" {
+ return fmt.Errorf("resumable uploads must set the Name parameter.")
+ }
+ } else if hasMedia_ {
+ req.Header.Set("Content-Type", ctype)
  }
- req.Header.Set("Content-Type", ctype)
  req.Header.Set("User-Agent", "google-api-go-client/0.5")
  res, err := c.s.client.Do(req)
  if err != nil {
@@ -165,6 +217,21 @@ func (c *MailInsertCall) Do() error {
  if err := googleapi.CheckResponse(res); err != nil {
  return err
  }
+ if c.protocol_ == "resumable" {
+ loc := res.Header.Get("Location")
+ rx := &googleapi.ResumableUpload{
+ Client: c.s.client,
+ URI: loc,
+ Media: c.resumable_,
+ MediaType: c.mediaType_,
+ ContentLength: c.resumable_.Size(),
+ Callback: progressUpdater_,
+ }
+ res, err = rx.Upload(c.ctx_)
+ if err != nil {
+ return err
+ }
+ }
  return nil
  // {
  // "description": "Insert Mail into Google's Gmail backends",

diff --git a/admin/reports_v1/admin-gen.go b/admin/reports_v1/admin-gen.go
@@ -14,6 +14,7 @@ import (
  "encoding/json"
  "errors"
  "fmt"
+ "golang.org/x/net/context"
  "google.golang.org/api/googleapi"
  "io"
  "net/http"
@@ -33,6 +34,7 @@ var _ = url.Parse
 var _ = googleapi.Version
 var _ = errors.New
 var _ = strings.Replace
+var _ = context.Background
 
 const apiId = "admin:reports_v1"
 const apiName = "admin"

diff --git a/adsense/v1.2/adsense-api.json b/adsense/v1.2/adsense-api.json
@@ -1,12 +1,12 @@
 {
  "kind": "discovery#restDescription",
- "etag": "\"ye6orv2F-1npMW3u9suM3a7C5Bo/EUOYPDRuvz7lgiNaz-i4liDZMRs\"",
+ "etag": "\"ye6orv2F-1npMW3u9suM3a7C5Bo/4sbh2CScTU4USzSvCohsjZ1YVxQ\"",
  "discoveryVersion": "v1",
  "id": "adsense:v1.2",
  "name": "adsense",
  "canonicalName": "AdSense",
  "version": "v1.2",
- "revision": "20141214",
+ "revision": "20141218",
  "title": "AdSense Management API",
  "description": "Gives AdSense publishers access to their inventory and the ability to generate reports",
  "ownerDomain": "google.com",

diff --git a/adsense/v1.2/adsense-gen.go b/adsense/v1.2/adsense-gen.go
@@ -14,6 +14,7 @@ import (
  "encoding/json"
  "errors"
  "fmt"
+ "golang.org/x/net/context"
  "google.golang.org/api/googleapi"
  "io"
  "net/http"
@@ -33,6 +34,7 @@ var _ = url.Parse
 var _ = googleapi.Version
 var _ = errors.New
 var _ = strings.Replace
+var _ = context.Background
 
 const apiId = "adsense:v1.2"
 const apiName = "adsense"

diff --git a/adsense/v1.3/adsense-api.json b/adsense/v1.3/adsense-api.json
@@ -1,12 +1,12 @@
 {
  "kind": "discovery#restDescription",
- "etag": "\"ye6orv2F-1npMW3u9suM3a7C5Bo/VH0tIFOeX_a2Unq7tkoTCRU1X60\"",
+ "etag": "\"ye6orv2F-1npMW3u9suM3a7C5Bo/gTf3J_Tcqy5jzBteF-x9VlOxF-c\"",
  "discoveryVersion": "v1",
  "id": "adsense:v1.3",
  "name": "adsense",
  "canonicalName": "AdSense",
  "version": "v1.3",
- "revision": "20141214",
+ "revision": "20141218",
  "title": "AdSense Management API",
  "description": "Gives AdSense publishers access to their inventory and the ability to generate reports",
  "ownerDomain": "google.com",

diff --git a/adsense/v1.3/adsense-gen.go b/adsense/v1.3/adsense-gen.go
@@ -14,6 +14,7 @@ import (
  "encoding/json"
  "errors"
  "fmt"
+ "golang.org/x/net/context"
  "google.golang.org/api/googleapi"
  "io"
  "net/http"
@@ -33,6 +34,7 @@ var _ = url.Parse
 var _ = googleapi.Version
 var _ = errors.New
 var _ = strings.Replace
+var _ = context.Background
 
 const apiId = "adsense:v1.3"
 const apiName = "adsense"

diff --git a/adsense/v1.4/adsense-api.json b/adsense/v1.4/adsense-api.json
@@ -1,12 +1,12 @@
 {
  "kind": "discovery#restDescription",
- "etag": "\"ye6orv2F-1npMW3u9suM3a7C5Bo/cSAJZw9ECrA-JzSWXpGjzWoNS_s\"",
+ "etag": "\"ye6orv2F-1npMW3u9suM3a7C5Bo/wiZb5dkU0x3zrMckXzFKcTNAp8w\"",
  "discoveryVersion": "v1",
  "id": "adsense:v1.4",
  "name": "adsense",
  "canonicalName": "AdSense",
  "version": "v1.4",
- "revision": "20141214",
+ "revision": "20141218",
  "title": "AdSense Management API",
  "description": "Gives AdSense publishers access to their inventory and the ability to generate reports",
  "ownerDomain": "google.com",

diff --git a/adsense/v1.4/adsense-gen.go b/adsense/v1.4/adsense-gen.go
@@ -14,6 +14,7 @@ import (
  "encoding/json"
  "errors"
  "fmt"
+ "golang.org/x/net/context"
  "google.golang.org/api/googleapi"
  "io"
  "net/http"
@@ -33,6 +34,7 @@ var _ = url.Parse
 var _ = googleapi.Version
 var _ = errors.New
 var _ = strings.Replace
+var _ = context.Background
 
 const apiId = "adsense:v1.4"
 const apiName = "adsense"