Fixes (#3394)

firebase · May 25, 2021 · 2bfecd2 · 2bfecd2
1 parent 95bee0e
commit 2bfecd2
Show file tree

Hide file tree

Showing 5 changed files with 170 additions and 51 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,3 +3,4 @@
 - Releases Firestore Emulator v1.12.0: supports clearing data partially.
 - Fixes manually setting download tokens in Storage Emulator. (#3396)
 - Fixes deleting custom metadata in Storage emulator. (#3385)
+- Fixes errors when calling makePublic() with Storage Emulator(#3394)
diff --git a/scripts/storage-emulator-integration/tests.ts b/scripts/storage-emulator-integration/tests.ts
@@ -298,6 +298,52 @@ describe("Storage emulator", () => {
  });
  });
 
+ describe("#makePublic()", () => {
+ it("should no-op", async () => {
+ const destination = "a/b";
+ await testBucket.upload(smallFilePath, { destination });
+ const [aclMetadata] = await testBucket.file(destination).makePublic();
+
+ const generation = aclMetadata.generation;
+ delete aclMetadata.generation;
+
+ expect(aclMetadata).to.deep.equal({
+ kind: "storage#objectAccessControl",
+ object: destination,
+ id: `${testBucket.name}/${destination}/${generation}/allUsers`,
+ selfLink: `${STORAGE_EMULATOR_HOST}/storage/v1/b/${
+ testBucket.name
+ }/o/${encodeURIComponent(destination)}/acl/allUsers`,
+ bucket: testBucket.name,
+ entity: "allUsers",
+ role: "READER",
+ etag: "someEtag",
+ });
+ });
+
+ it("should not interfere with downloading of bytes via public URL", async () => {
+ const destination = "a/b";
+ await testBucket.upload(smallFilePath, { destination });
+ await testBucket.file(destination).makePublic();
+
+ const publicLink = `${STORAGE_EMULATOR_HOST}/${testBucket.name}/${destination}`;
+
+ const requestClient = TEST_CONFIG.useProductionServers ? https : http;
+ await new Promise((resolve, reject) => {
+ requestClient.get(publicLink, {}, (response) => {
+ const data: any = [];
+ response
+ .on("data", (chunk) => data.push(chunk))
+ .on("end", () => {
+ expect(Buffer.concat(data).length).to.equal(SMALL_FILE_SIZE);
+ })
+ .on("close", resolve)
+ .on("error", reject);
+ });
+ });
+ });
+ });
+
  describe("#getMetadata()", () => {
  it("should throw on non-existing file", async () => {
  let err: any;

diff --git a/src/emulator/storage/apis/gcloud.ts b/src/emulator/storage/apis/gcloud.ts
@@ -1,9 +1,16 @@
 import { Router } from "express";
 import { gunzipSync } from "zlib";
 import { Emulators } from "../../types";
-import { CloudStorageObjectMetadata } from "../metadata";
+import {
+ CloudStorageObjectAccessControlMetadata,
+ CloudStorageObjectMetadata,
+ StoredFileMetadata,
+} from "../metadata";
 import { EmulatorRegistry } from "../../registry";
 import { StorageEmulator } from "../index";
+import { EmulatorLogger } from "../../emulatorLogger";
+import { StorageLayer } from "../files";
+import type { Request, Response } from "express";
 
 /**
  * @param emulator
@@ -14,19 +21,19 @@ export function createCloudEndpoints(emulator: StorageEmulator): Router {
  const gcloudStorageAPI = Router();
  const { storageLayer } = emulator;
 
+ // Automatically create a bucket for any route which uses a bucket
+ gcloudStorageAPI.use(/.*\/b\/(.+?)\/.*/, (req, res, next) => {
+ storageLayer.createBucket(req.params[0]);
+ next();
+ });
+
  gcloudStorageAPI.get("/b", (req, res) => {
  res.json({
  kind: "storage#buckets",
  items: storageLayer.listBuckets(),
  });
  });
 
- // Automatically create a bucket for any route which uses a bucket
- gcloudStorageAPI.use(/.*\/b\/(.+?)\/.*/, (req, res, next) => {
- storageLayer.createBucket(req.params[0]);
- next();
- });
-
  gcloudStorageAPI.get(
  ["/b/:bucketId/o/:objectId", "/download/storage/v1/b/:bucketId/o/:objectId"],
  (req, res) => {
@@ -38,40 +45,7 @@ export function createCloudEndpoints(emulator: StorageEmulator): Router {
  }
 
  if (req.query.alt == "media") {
- let data = storageLayer.getBytes(req.params.bucketId, req.params.objectId);
- if (!data) {
- res.sendStatus(404);
- return;
- }
-
- const isGZipped = md.contentEncoding == "gzip";
- if (isGZipped) {
- data = gunzipSync(data);
- }
-
- res.setHeader("Accept-Ranges", "bytes");
- res.setHeader("Content-Type", md.contentType);
- res.setHeader("Content-Disposition", md.contentDisposition);
- res.setHeader("Content-Encoding", "identity");
-
- const byteRange = [...(req.header("range") || "").split("bytes="), "", ""];
-
- const [rangeStart, rangeEnd] = byteRange[1].split("-");
-
- if (rangeStart) {
- const range = {
- start: parseInt(rangeStart),
- end: rangeEnd ? parseInt(rangeEnd) : data.byteLength,
- };
- res.setHeader(
- "Content-Range",
- `bytes ${range.start}-${range.end - 1}/${data.byteLength}`
- );
- res.status(206).end(data.slice(range.start, range.end));
- } else {
- res.end(data);
- }
- return;
+ return sendFileBytes(md, storageLayer, req, res);
  }
 
  const outgoingMd = new CloudStorageObjectMetadata(md);
@@ -165,6 +139,39 @@ export function createCloudEndpoints(emulator: StorageEmulator): Router {
  res.status(200).json(new CloudStorageObjectMetadata(finalizedUpload.file.metadata)).send();
  });
 
+ gcloudStorageAPI.post("/b/:bucketId/o/:objectId/acl", (req, res) => {
+ // TODO(abehaskins) Link to a doc with more info
+ EmulatorLogger.forEmulator(Emulators.STORAGE).log(
+ "WARN_ONCE",
+ "Cloud Storage ACLs are not supported in the Storage Emulator. All related methods will succeed, but have no effect."
+ );
+ const md = storageLayer.getMetadata(req.params.bucketId, req.params.objectId);
+
+ if (!md) {
+ res.sendStatus(404);
+ return;
+ }
+
+ // We do an empty update to step metageneration forward;
+ md.update({});
+
+ res
+ .json({
+ kind: "storage#objectAccessControl",
+ object: md.name,
+ id: `${req.params.bucketId}/${md.name}/${md.generation}/allUsers`,
+ selfLink: `http://${EmulatorRegistry.getInfo(Emulators.STORAGE)?.host}:${
+ EmulatorRegistry.getInfo(Emulators.STORAGE)?.port
+ }/storage/v1/b/${md.bucket}/o/${encodeURIComponent(md.name)}/acl/allUsers`,
+ bucket: md.bucket,
+ entity: req.body.entity,
+ role: req.body.role,
+ etag: "someEtag",
+ generation: md.generation.toString(),
+ } as CloudStorageObjectAccessControlMetadata)
+ .status(200);
+ });
+
  gcloudStorageAPI.post("/upload/storage/v1/b/:bucketId/o", (req, res) => {
  if (!req.query.name) {
  res.sendStatus(400);
@@ -250,5 +257,65 @@ export function createCloudEndpoints(emulator: StorageEmulator): Router {
  return;
  });
 
+ gcloudStorageAPI.get("/:bucketId/:objectId(**)", (req, res) => {
+ const md = storageLayer.getMetadata(req.params.bucketId, req.params.objectId);
+
+ if (!md) {
+ res.sendStatus(404);
+ return;
+ }
+
+ return sendFileBytes(md, storageLayer, req, res);
+ });
+
+ gcloudStorageAPI.all("/**", (req, res) => {
+ if (true || process.env.STORAGE_EMULATOR_DEBUG) {
+ console.table(req.headers);
+ console.log(req.method, req.url);
+ res.json("endpoint not implemented");
+ } else {
+ res.sendStatus(501);
+ }
+ });
+
  return gcloudStorageAPI;
 }
+
+function sendFileBytes(
+ md: StoredFileMetadata,
+ storageLayer: StorageLayer,
+ req: Request,
+ res: Response
+) {
+ let data = storageLayer.getBytes(req.params.bucketId, req.params.objectId);
+ if (!data) {
+ res.sendStatus(404);
+ return;
+ }
+
+ const isGZipped = md.contentEncoding == "gzip";
+ if (isGZipped) {
+ data = gunzipSync(data);
+ }
+
+ res.setHeader("Accept-Ranges", "bytes");
+ res.setHeader("Content-Type", md.contentType);
+ res.setHeader("Content-Disposition", md.contentDisposition);
+ res.setHeader("Content-Encoding", "identity");
+
+ const byteRange = [...(req.header("range") || "").split("bytes="), "", ""];
+
+ const [rangeStart, rangeEnd] = byteRange[1].split("-");
+
+ if (rangeStart) {
+ const range = {
+ start: parseInt(rangeStart),
+ end: rangeEnd ? parseInt(rangeEnd) : data.byteLength,
+ };
+ res.setHeader("Content-Range", `bytes ${range.start}-${range.end - 1}/${data.byteLength}`);
+ res.status(206).end(data.slice(range.start, range.end));
+ } else {
+ res.end(data);
+ }
+ return;
+}
diff --git a/src/emulator/storage/metadata.ts b/src/emulator/storage/metadata.ts
@@ -345,6 +345,21 @@ export class CloudStorageBucketMetadata {
  }
 }
 
+export class CloudStorageObjectAccessControlMetadata {
+ kind = "storage#objectAccessControl";
+
+ constructor(
+ public object: string,
+ public generation: string,
+ public selfLink: string,
+ public id: string,
+ public role: string,
+ public entity: string,
+ public bucket: string,
+ public etag: string
+ ) {}
+}
+
 export class CloudStorageObjectMetadata {
  kind = "#storage#object";
  name: string;

diff --git a/src/emulator/storage/server.ts b/src/emulator/storage/server.ts
@@ -114,15 +114,5 @@ export function createApp(
  app.use("/v0", createFirebaseEndpoints(emulator));
  app.use("/", createCloudEndpoints(emulator));
 
- app.all("**", (req, res) => {
- if (process.env.STORAGE_EMULATOR_DEBUG) {
- console.table(req.headers);
- console.log(req.method, req.url);
- res.json("endpoint not implemented");
- } else {
- res.sendStatus(404);
- }
- });
-
  return Promise.resolve(app);
 }