Add command to prune unused secrets (#4108)

Each active secret version cost money. To help save cost on using Secret Manager, we add `functions:secrets:prune` command which: 1) Looks up all secret versions from secrets marked with label "firebase-managed". All secrets created using the Firebase CLI will have this label. 2) Look up all secret bindings for CF3 function instance. 3) Figure out which secret version isn't currently being used. Since destroying a secret version is irrevocable and immediately breaking for clients that depend on it, we will always ask for a confirmation for the destroy operations (and not support -f flag). Note that we now query `v1` of Secret Manager since `v1beta` does not offer filtering by labels.
firebase · taeold · Feb 10, 2022 · Jan 27, 2022 · Jan 31, 2022 · Jan 31, 2022
commit 3310fdc21a5f6988d1b1d28765b3e74877bbd1b9
diff --git a/src/commands/functions-secrets-prune.ts b/src/commands/functions-secrets-prune.ts
@@ -0,0 +1,68 @@
+import * as args from "../deploy/functions/args";
+import * as backend from "../deploy/functions/backend";
+import { Command } from "../command";
+import { Options } from "../options";
+import { needProjectId, needProjectNumber } from "../projectUtils";
+import { pruneSecrets } from "../functions/secrets";
+import { requirePermissions } from "../requirePermissions";
+import { isFirebaseManaged } from "../deploymentTool";
+import { logBullet, logSuccess } from "../utils";
+import { promptOnce } from "../prompt";
+import { destroySecretVersion } from "../gcp/secretManager";
+
+export default new Command("functions:secrets:prune")
+ .description("Destroys unused secrets")
+ .before(requirePermissions, [
+ "cloudfunctions.functions.list",
+ "secretmanager.secrets.list",
+ "secretmanager.versions.list",
+ "secretmanager.versions.destroy",
+ ])
+ .action(async (options: Options) => {
+ const projectNumber = await needProjectNumber(options);
+ const projectId = needProjectId(options);
+
+ logBullet("Loading secrets...");
+
+ const haveBackend = await backend.existingBackend({ projectId } as args.Context);
+ const haveEndpoints = backend
+ .allEndpoints(haveBackend)
+ .filter((e) => isFirebaseManaged(e.labels || []));
+
+ const pruned = await pruneSecrets({ projectNumber, projectId }, haveEndpoints);
+
+ if (pruned.length === 0) {
+ logBullet("All secrets are in use. Nothing to prune today.");
+ return;
+ }
+
+ // prompt to get them all deleted
+ logBullet(
+ `Found ${pruned.length} unused active secret versions:\n\t` +
+ pruned.map((sv) => `${sv.secret}@${sv.version}`).join("\n\t")
+ );
+
+ const confirm = await promptOnce(
+ {
+ name: "destroy",
+ type: "confirm",
+ default: true,
+ message: `Do you want to destroy unused secret versions?`,
+ },
+ options
+ );
+
+ if (!confirm) {
+ logBullet(
+ "Run the following commands to destroy each unused secret version:\n\t" +
+ pruned
+ .map((sv) => `firebase functions:secrets:destroy ${sv.secret}@${sv.version}`)
+ .join("\n\t")
+ );
+ return;
+ }
+
+ await Promise.all(pruned.map((sv) => destroySecretVersion(projectId, sv.secret, sv.version)));
+
+ logSuccess("Destroyed all unused secrets!");
+ });
diff --git a/src/commands/index.js b/src/commands/index.js
@@ -112,6 +112,7 @@ module.exports = function (client) {
  client.functions.secrets.access = loadCommand("functions-secrets-access");
  client.functions.secrets.destroy = loadCommand("functions-secrets-destroy");
  client.functions.secrets.get = loadCommand("functions-secrets-get");
+ client.functions.secrets.prune = loadCommand("functions-secrets-prune");
  client.functions.secrets.set = loadCommand("functions-secrets-set");
  client.help = loadCommand("help");
  client.hosting = {};

diff --git a/src/deploy/functions/backend.ts b/src/deploy/functions/backend.ts
@@ -169,6 +169,7 @@ export interface TargetIds {
 export interface SecretEnvVar {
  key: string;
  secret: string;
+ projectId: string;
 
  // Internal use only. Users cannot pin secret to a specific version.
  version?: string;

diff --git a/src/deploy/functions/runtimes/node/parseTriggers.ts b/src/deploy/functions/runtimes/node/parseTriggers.ts
@@ -226,6 +226,7 @@ export function addResourcesToBackend(
  for (const secret of annotation.secrets) {
  const secretEnv: backend.SecretEnvVar = {
  secret,
+ projectId,
  key: secret,
  };
  secretEnvs.push(secretEnv);

diff --git a/src/deploy/functions/validate.ts b/src/deploy/functions/validate.ts
@@ -7,6 +7,7 @@ import { logger } from "../../logger";
 import * as fsutils from "../../fsutils";
 import * as backend from "./backend";
 import * as utils from "../../utils";
+import * as secrets from "../../functions/secrets";
 
 /** Validate that the configuration for endpoints are valid. */
 export function endpointsAreValid(wantBackend: backend.Backend): void {
@@ -126,10 +127,8 @@ function validatePlatformTargets(endpoints: backend.Endpoint[]) {
  */
 async function validateSecretVersions(projectId: string, endpoints: backend.Endpoint[]) {
  const toResolve: Set<string> = new Set();
- for (const e of endpoints) {
- for (const s of e.secretEnvironmentVariables! || []) {
- toResolve.add(s.secret);
- }
+ for (const s of secrets.of(endpoints)) {
+ toResolve.add(s.secret);
  }
 
  const results = await utils.allSettled(
@@ -164,14 +163,12 @@ async function validateSecretVersions(projectId: string, endpoints: backend.Endp
  }
 
  // Fill in versions.
- for (const e of endpoints) {
- for (const s of e.secretEnvironmentVariables! || []) {
- s.version = secretVersions[s.secret].versionId;
- if (!s.version) {
- throw new FirebaseError(
- "Secret version is unexpectedly undefined. This should never happen."
- );
- }
+ for (const s of secrets.of(endpoints)) {
+ s.version = secretVersions[s.secret].versionId;
+ if (!s.version) {
+ throw new FirebaseError(
+ "Secret version is unexpectedly undefined. This should never happen."
+ );
  }
  }
 }
diff --git a/src/functions/secrets.ts b/src/functions/secrets.ts
@@ -1,4 +1,14 @@
-import { createSecret, getSecret, patchSecret, Secret } from "../gcp/secretManager";
+import * as backend from "../deploy/functions/backend";
+import {
+ createSecret,
+ getSecret,
+ getSecretVersion,
+ listSecrets,
+ listSecretVersions,
+ parseSecretResourceName,
+ patchSecret,
+ Secret,
+} from "../gcp/secretManager";
 import { Options } from "../options";
 import { FirebaseError } from "../error";
 import { logWarning } from "../utils";
@@ -102,3 +112,59 @@ export async function ensureSecret(
  }
  return await createSecret(projectId, name, labels());
 }
+
+/**
+ * Collects all secret environment variables of endpoints.
+ */
+export function of(endpoints: backend.Endpoint[]): backend.SecretEnvVar[] {
+ return endpoints.reduce(
+ (envs, endpoint) => [...envs, ...(endpoint.secretEnvironmentVariables || [])],
+ [] as backend.SecretEnvVar[]
+ );
+}
+
+/**
+ * Returns all secret versions from Firebase managed secrets unused in the given list of endpoints.
+ */
+export async function pruneSecrets(
+ projectInfo: { projectNumber: string; projectId: string },
+ endpoints: backend.Endpoint[]
+): Promise<Required<backend.SecretEnvVar>[]> {
+ const { projectId, projectNumber } = projectInfo;
+ const pruneKey = (name: string, version: string) => `${name}@${version}`;
+ const prunedSecrets: Set<string> = new Set();
+
+ // Collect all Firebase managed secret versions
+ const haveSecrets = await listSecrets(projectId, `labels.${FIREBASE_MANGED}=true`);
+ for (const secret of haveSecrets) {
+ const versions = await listSecretVersions(projectId, secret.name, `state: ENABLED`);
+ for (const version of versions) {
+ prunedSecrets.add(pruneKey(secret.name, version.versionId));
+ }
+ }
+
+ // Prune all project-scoped secrets in use.
+ const sevs = of(endpoints).filter(
+ (sev) => sev.projectId === projectId || sev.projectId === projectNumber
+ );
+ for (const sev of sevs) {
+ let name = sev.secret;
+ if (name.includes("/")) {
+ const secret = parseSecretResourceName(name);
+ name = secret.name;
+ }
+
+ let version = sev.version;
+ if (version === "latest") {
+ // We need to figure out what "latest" resolves to.
+ const resolved = await getSecretVersion(projectId, name, version);
+ version = resolved.versionId;
+ }
+
+ prunedSecrets.delete(pruneKey(name, version!));
+ }
+
+ return Array.from(prunedSecrets)
+ .map((key) => key.split("@"))
+ .map(([secret, version]) => ({ projectId, version, secret, key: secret }));
+}
diff --git a/src/gcp/cloudfunctions.ts b/src/gcp/cloudfunctions.ts
@@ -517,6 +517,7 @@ export function endpointFromFunction(gcfFunction: CloudFunction): backend.Endpoi
  "ingressSettings",
  "labels",
  "environmentVariables",
+ "secretEnvironmentVariables",
  "sourceUploadUrl"
  );
 

diff --git a/src/gcp/secretManager.ts b/src/gcp/secretManager.ts
@@ -60,18 +60,10 @@ interface AccessSecretVersionResponse {
  };
 }
 
-const API_VERSION = "v1beta1";
+const API_VERSION = "v1";
 
 const client = new Client({ urlPrefix: secretManagerOrigin, apiVersion: API_VERSION });
 
-/**
- * Returns all secret resources of given project.
- */
-export async function listSecrets(projectId: string): Promise<Secret[]> {
- const listRes = await client.get<{ secrets: Secret[] }>(`projects/${projectId}/secrets`);
- return listRes.body.secrets.map((s: any) => parseSecretResourceName(s.name));
-}
-
 /**
  * Returns secret resource of given name in the project.
  */
@@ -82,23 +74,60 @@ export async function getSecret(projectId: string, name: string): Promise<Secret
  return secret;
 }
 
+/**
+ * Lists all secret resources associated with a project.
+ */
+export async function listSecrets(projectId: string, filter?: string): Promise<Secret[]> {
+ type Response = { secrets: Secret[]; nextPageToken?: string };
+ const secrets: Secret[] = [];
+ const path = `projects/${projectId}/secrets`;
+ const baseOpts = filter ? { queryParams: { filter } } : {};
+
+ let pageToken = "";
+ while (true) {
+ const opts =
+ pageToken === ""
+ ? baseOpts
+ : { ...baseOpts, queryParams: { ...baseOpts?.queryParams, pageToken } };
+ const res = await client.get<Response>(path, opts);
+
+ for (const s of res.body.secrets) {
+ secrets.push({
+ ...parseSecretResourceName(s.name),
+ labels: s.labels ?? {},
+ });
+ }
+
+ if (!res.body.nextPageToken) {
+ break;
+ }
+ pageToken = res.body.nextPageToken;
+ }
+ return secrets;
+}
+
 /**
  * List all secret versions associated with a secret.
  */
 export async function listSecretVersions(
  projectId: string,
- name: string
+ name: string,
+ filter?: string
 ): Promise<Required<SecretVersion[]>> {
  type Response = { versions: SecretVersionResponse[]; nextPageToken?: string };
  const secrets: Required<SecretVersion[]> = [];
  const path = `projects/${projectId}/secrets/${name}/versions`;
+ const baseOpts = filter ? { queryParams: { filter } } : {};
 
  let pageToken = "";
  while (true) {
- const opts = pageToken == "" ? {} : { queryParams: { pageToken } };
+ const opts =
+ pageToken === ""
+ ? baseOpts
+ : { ...baseOpts, queryParams: { ...baseOpts?.queryParams, pageToken } };
  const res = await client.get<Response>(path, opts);
 
- for (const s of res.body.versions) {
+ for (const s of res.body.versions || []) {
  secrets.push({
  ...parseSecretVersionResourceName(s.name),
  state: s.state,

diff --git a/src/test/deploy/functions/ensure.spec.ts b/src/test/deploy/functions/ensure.spec.ts
@@ -152,11 +152,13 @@ describe("ensureSecretAccess", () => {
 
  const projectId = "project-0";
  const secret0: backend.SecretEnvVar = {
+ projectId: "project",
  key: "MY_SECRET_0",
  secret: "MY_SECRET_0",
  version: "2",
  };
  const secret1: backend.SecretEnvVar = {
+ projectId: "project",
  key: "ANOTHER_SECRET",
  secret: "ANOTHER_SECRET",
  version: "1",

diff --git a/src/test/deploy/functions/runtimes/node/parseTriggers.spec.ts b/src/test/deploy/functions/runtimes/node/parseTriggers.spec.ts
@@ -321,6 +321,7 @@ describe("addResourcesToBackend", () => {
  httpsTrigger: {},
  secretEnvironmentVariables: [
  {
+ projectId: "project",
  secret: "MY_SECRET",
  key: "MY_SECRET",
  },

diff --git a/src/test/deploy/functions/validate.spec.ts b/src/test/deploy/functions/validate.spec.ts
@@ -246,6 +246,7 @@ describe("validate", () => {
  platform: "gcfv2",
  secretEnvironmentVariables: [
  {
+ projectId: project,
  secret: "MY_SECRET",
  key: "MY_SECRET",
  },
@@ -263,6 +264,7 @@ describe("validate", () => {
  platform: "gcfv1",
  secretEnvironmentVariables: [
  {
+ projectId: project,
  secret: "MY_SECRET",
  key: "MY_SECRET",
  },
@@ -283,6 +285,7 @@ describe("validate", () => {
  platform: "gcfv1",
  secretEnvironmentVariables: [
  {
+ projectId: project,
  secret: "MY_SECRET",
  key: "MY_SECRET",
  },
@@ -303,6 +306,7 @@ describe("validate", () => {
  platform: "gcfv1",
  secretEnvironmentVariables: [
  {
+ projectId: project,
  secret: "MY_SECRET",
  key: "MY_SECRET",
  },

diff --git a/src/test/extensions/secretUtils.spec.ts b/src/test/extensions/secretUtils.spec.ts
@@ -60,13 +60,13 @@ describe("secretsUtils", () => {
  describe("getManagedSecrets", () => {
  it("only returns secrets that have labels set", async () => {
  nock(api.secretManagerOrigin)
- .get(`/v1beta1/projects/${PROJECT_ID}/secrets/secret1`)
+ .get(`/v1/projects/${PROJECT_ID}/secrets/secret1`)
  .reply(200, {
  name: `projects/${PROJECT_ID}/secrets/secret1`,
  labels: { "firebase-extensions-managed": "true" },
  });
  nock(api.secretManagerOrigin)
- .get(`/v1beta1/projects/${PROJECT_ID}/secrets/secret2`)
+ .get(`/v1/projects/${PROJECT_ID}/secrets/secret2`)
  .reply(200, {
  name: `projects/${PROJECT_ID}/secrets/secret2`,
  }); // no labels