Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Major refactor of functions deploy #3132

Merged
merged 37 commits into from
Feb 16, 2021
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
06b62a1
Adding onPoll option to operation-poller (#3046)
joehan Jan 19, 2021
6258dd5
Typescriptify functionsDeployHelper (#3059)
joehan Jan 20, 2021
ec7d079
Typescriptifying gcp.cloudfunctions (#3060)
joehan Jan 20, 2021
f583ef6
Typescriptifying functionsConfig (#3063)
joehan Jan 21, 2021
08b9d56
Typescriptifying deploymentTool (#3061)
joehan Jan 21, 2021
b4944a4
Refactoring prepare stage of functions deploy (#3067)
joehan Jan 21, 2021
3be0dca
refactoring release step of functions deploy to use typescript
joehan Jan 21, 2021
e0e703e
Adding logic to build regional deployments
joehan Jan 24, 2021
046c7d7
Implementing createDeploymentPlan
joehan Jan 26, 2021
b876523
First round of PR feedback, removing most usages of lodash
joehan Jan 28, 2021
9e0e6e9
moving function prompts into their own file
joehan Jan 28, 2021
2a3b547
seperating out a bunch of code from functionsDeployHelper
joehan Jan 28, 2021
51f2395
Resolves merge conflicts
joehan Jan 28, 2021
30cc0e9
refactoring release step of functions deploy to use typescript (#3071)
joehan Feb 1, 2021
6916000
Implements core logic of running function deploys
joehan Feb 1, 2021
3c8d4a0
Typescriptifying prepareFunctionsUpload (#3064)
joehan Feb 1, 2021
11956fa
Implementing createDeploymentPlan (#3081)
joehan Feb 1, 2021
85d0afe
adding timing and logs for deployments
joehan Feb 2, 2021
00b1989
cleaning up unused code
joehan Feb 2, 2021
397d7c4
Fixing some things that were broken while merging
joehan Feb 3, 2021
21f4906
Fixing up the order of wait and close to ensure that queue promsies a…
joehan Feb 4, 2021
3b3edbd
Format and clean up typos
joehan Feb 4, 2021
e428bcb
refactoring error handling to be cleaner
joehan Feb 5, 2021
4c8e2fb
cleaning up extera newlines
joehan Feb 8, 2021
7f48130
first round of pr fixes
joehan Feb 9, 2021
39a7e86
Readding some changes that I accidenttally wiped out during a merge
joehan Feb 9, 2021
1366955
Switching name to id where appropriate
joehan Feb 9, 2021
7513229
fixing another bug caused by functionName vs Id
joehan Feb 9, 2021
8d3d82d
Merge pull request #3107 from firebase/jh-execute-deployment-plans
joehan Feb 9, 2021
6d2260e
Refactor functions-delete (#3110)
joehan Feb 9, 2021
42e6c15
Cleaning up error reporting
joehan Feb 10, 2021
e4ce126
Merge remote-tracking branch 'public/master' into jh-functions-refactor
joehan Feb 10, 2021
12a48ea
Merge remote-tracking branch 'public/master' into jh-functions-refactor
joehan Feb 11, 2021
7cfe9d9
Implement validation for changing trigger types, and fixes from bug b…
joehan Feb 12, 2021
5eb08bd
Merge branch 'master' into jh-functions-refactor
joehan Feb 12, 2021
5ca6bbf
Merge branch 'master' into jh-functions-refactor
joehan Feb 16, 2021
344b674
fixes package.json
joehan Feb 16, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Prev Previous commit
Next Next commit
refactoring error handling to be cleaner
  • Loading branch information
joehan committed Feb 5, 2021
commit e428bcb75b9293616c86a9b15a137a5035b42ef6
29 changes: 5 additions & 24 deletions src/deploy/functions/release.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,8 @@ export async function release(context: any, options: any, payload: any) {
);
if (shouldDeleteFunctions) {
for (const fnName of fullDeployment.functionsToDelete) {
cloudFunctionsQueue
.run(tasks.deleteFunctionTask(taskParams, fnName))
.then(() => {
helper.printSuccess(fnName, "delete");
})
.catch((err) => {
errorHandler.record("error", fnName, "delete", err.message || "");
});
const task = tasks.deleteFunctionTask(taskParams, fnName);
cloudFunctionsQueue.run(task);
}
} else {
// If we shouldn't delete functions, don't clean up their schedules either
Expand All @@ -78,25 +72,12 @@ export async function release(context: any, options: any, payload: any) {
// Add scheduler creates and updates to their queue.
for (const fn of regionalDeployment.schedulesToUpsert) {
const task = tasks.upsertScheduleTask(taskParams, fn, appEngineLocation);
schedulerQueue
.run(task)
.then(() => {
helper.printSuccess(fn.name, "upsert schedule");
})
.catch((err) => {
errorHandler.record("error", fn.name, "upsert schedule", err.message || "");
});
schedulerQueue.run(task);
}
}
for (const fnName of fullDeployment.schedulesToDelete) {
schedulerQueue
.run(tasks.deleteScheduleTask(fnName, appEngineLocation))
.then(() => {
helper.printSuccess(fnName, "delete schedule");
})
.catch((err) => {
errorHandler.record("error", fnName, "delete schedule", err.message || "");
});
const task = tasks.deleteScheduleTask(taskParams, fnName, appEngineLocation);
schedulerQueue.run(task);
}

// Once everything has been added to queues, starting processing.
Expand Down
249 changes: 123 additions & 126 deletions src/deploy/functions/tasks.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ export function createFunctionTask(
params: TaskParams,
fn: CloudFunctionTrigger,
onPoll?: (op: any) => any
): () => Promise<CloudFunctionTrigger> {
): () => Promise<CloudFunctionTrigger | void> {
return async () => {
utils.logBullet(
clc.bold.cyan("functions: ") +
Expand All @@ -47,56 +47,61 @@ export function createFunctionTask(
);
params.timer.startTimer(fn.name, "create");
const eventType = fn.eventTrigger ? fn.eventTrigger.eventType : "https";
const createRes = await cloudfunctions.createFunction({
projectId: params.projectId,
region: helper.getRegion(fn.name),
eventType: eventType,
functionName: helper.getFunctionName(fn.name),
entryPoint: fn.entryPoint,
trigger: helper.getFunctionTrigger(fn),
labels: Object.assign({}, deploymentTool.labels(), fn.labels),
sourceUploadUrl: params.sourceUrl,
sourceToken: params.sourceToken,
runtime: params.runtime,
availableMemoryMb: fn.availableMemoryMb,
timeout: fn.timeout,
maxInstances: fn.maxInstances,
environmentVariables: fn.environmentVariables,
vpcConnector: fn.vpcConnector,
vpcConnectorEgressSettings: fn.vpcConnectorEgressSettings,
serviceAccountEmail: fn.serviceAccountEmail,
});
const pollerOptions: OperationPollerOptions = Object.assign(
{
pollerName: `create-${fn.name}`,
operationResourceName: createRes.name,
onPoll,
},
defaultPollerOptions
);
const operationResult = await pollOperation<CloudFunctionTrigger>(pollerOptions);
if (eventType === "https") {
try {
await cloudfunctions.setIamPolicy({
functionName: fn.name,
projectId: params.projectId,
region: helper.getRegion(fn.name),
policy: cloudfunctions.DEFAULT_PUBLIC_POLICY,
});
} catch (err) {
params.errorHandler.record("warning", fn.name, "make public", err.message);
try {
const createRes = await cloudfunctions.createFunction({
projectId: params.projectId,
region: helper.getRegion(fn.name),
eventType: eventType,
functionName: helper.getFunctionName(fn.name),
entryPoint: fn.entryPoint,
trigger: helper.getFunctionTrigger(fn),
labels: Object.assign({}, deploymentTool.labels(), fn.labels),
sourceUploadUrl: params.sourceUrl,
sourceToken: params.sourceToken,
runtime: params.runtime,
availableMemoryMb: fn.availableMemoryMb,
timeout: fn.timeout,
maxInstances: fn.maxInstances,
environmentVariables: fn.environmentVariables,
vpcConnector: fn.vpcConnector,
vpcConnectorEgressSettings: fn.vpcConnectorEgressSettings,
serviceAccountEmail: fn.serviceAccountEmail,
});
const pollerOptions: OperationPollerOptions = Object.assign(
{
pollerName: `create-${fn.name}`,
operationResourceName: createRes.name,
onPoll,
},
defaultPollerOptions
);
const operationResult = await pollOperation<CloudFunctionTrigger>(pollerOptions);
if (eventType === "https") {
try {
await cloudfunctions.setIamPolicy({
functionName: fn.name,
projectId: params.projectId,
region: helper.getRegion(fn.name),
policy: cloudfunctions.DEFAULT_PUBLIC_POLICY,
});
} catch (err) {
params.errorHandler.record("warning", fn.name, "make public", err.message);
}
}
params.timer.endTimer(fn.name);
helper.printSuccess(fn.name, "create");
return operationResult;
} catch (err) {
params.errorHandler.record("error", fn.name, "create", err.message || "");
}
params.timer.endTimer(fn.name);
return operationResult;
};
}

export function updateFunctionTask(
params: TaskParams,
fn: CloudFunctionTrigger,
onPoll?: (op: any) => any
): () => Promise<CloudFunctionTrigger> {
): () => Promise<CloudFunctionTrigger | void> {
return async () => {
utils.logBullet(
clc.bold.cyan("functions: ") +
Expand All @@ -108,36 +113,41 @@ export function updateFunctionTask(
);
params.timer.startTimer(fn.name, "update");
const eventType = fn.eventTrigger ? fn.eventTrigger.eventType : "https";
const updateRes = await cloudfunctions.updateFunction({
projectId: params.projectId,
region: helper.getRegion(fn.name),
eventType: eventType,
functionName: helper.getFunctionName(fn.name),
entryPoint: fn.entryPoint,
trigger: helper.getFunctionTrigger(fn),
labels: Object.assign({}, deploymentTool.labels(), fn.labels),
sourceUploadUrl: params.sourceUrl,
sourceToken: params.sourceToken,
runtime: params.runtime,
availableMemoryMb: fn.availableMemoryMb,
timeout: fn.timeout,
maxInstances: fn.maxInstances,
environmentVariables: fn.environmentVariables,
vpcConnector: fn.vpcConnector,
vpcConnectorEgressSettings: fn.vpcConnectorEgressSettings,
serviceAccountEmail: fn.serviceAccountEmail,
});
const pollerOptions: OperationPollerOptions = Object.assign(
{
pollerName: `update-${fn.name}`,
operationResourceName: updateRes.name,
onPoll,
},
defaultPollerOptions
);
const operationResult = await pollOperation<CloudFunctionTrigger>(pollerOptions);
params.timer.endTimer(fn.name);
return operationResult;
try {
const updateRes = await cloudfunctions.updateFunction({
projectId: params.projectId,
region: helper.getRegion(fn.name),
eventType: eventType,
functionName: helper.getFunctionName(fn.name),
entryPoint: fn.entryPoint,
trigger: helper.getFunctionTrigger(fn),
labels: Object.assign({}, deploymentTool.labels(), fn.labels),
sourceUploadUrl: params.sourceUrl,
sourceToken: params.sourceToken,
runtime: params.runtime,
availableMemoryMb: fn.availableMemoryMb,
timeout: fn.timeout,
maxInstances: fn.maxInstances,
environmentVariables: fn.environmentVariables,
vpcConnector: fn.vpcConnector,
vpcConnectorEgressSettings: fn.vpcConnectorEgressSettings,
serviceAccountEmail: fn.serviceAccountEmail,
});
const pollerOptions: OperationPollerOptions = Object.assign(
{
pollerName: `update-${fn.name}`,
operationResourceName: updateRes.name,
onPoll,
},
defaultPollerOptions
);
const operationResult = await pollOperation<CloudFunctionTrigger>(pollerOptions);
params.timer.endTimer(fn.name);
helper.printSuccess(fn.name, "update");
return operationResult;
} catch (err) {
params.errorHandler.record("error", fn.name, "update", err.message || "");
}
};
}

Expand All @@ -150,19 +160,24 @@ export function deleteFunctionTask(params: TaskParams, fnName: string) {
"..."
);
params.timer.startTimer(fnName, "delete");
const deleteRes = await cloudfunctions.deleteFunction({
functionName: fnName,
});
const pollerOptions: OperationPollerOptions = Object.assign(
{
pollerName: `delete-${fnName}`,
operationResourceName: deleteRes.name,
},
defaultPollerOptions
);
const operationResult = await pollOperation<void>(pollerOptions);
params.timer.endTimer(fnName);
return operationResult;
try {
const deleteRes = await cloudfunctions.deleteFunction({
functionName: fnName,
});
const pollerOptions: OperationPollerOptions = Object.assign(
{
pollerName: `delete-${fnName}`,
operationResourceName: deleteRes.name,
},
defaultPollerOptions
);
const operationResult = await pollOperation<void>(pollerOptions);
params.timer.endTimer(fnName);
helper.printSuccess(fnName, "delete");
return operationResult;
} catch (err) {
params.errorHandler.record("error", fnName, "delete", err.message || "");
}
};
}

Expand All @@ -173,31 +188,41 @@ export function upsertScheduleTask(
): () => Promise<any> {
return async () => {
const job = helper.toJob(fn, appEngineLocation, params.projectId);
return cloudscheduler.createOrReplaceJob(job);
try {
await cloudscheduler.createOrReplaceJob(job);
helper.printSuccess(fn.name, "upsert schedule");
} catch (err) {
params.errorHandler.record("error", fn.name, "upsert schedule", err.message || "");
}
};
}

export function deleteScheduleTask(fnName: string, appEngineLocation: string): () => Promise<void> {
export function deleteScheduleTask(
params: TaskParams,
fnName: string,
appEngineLocation: string
): () => Promise<void> {
return async () => {
const jobName = helper.getScheduleName(fnName, appEngineLocation);
const topicName = helper.getTopicName(fnName);
try {
await cloudscheduler.deleteJob(jobName);
} catch (err) {
// If the job has already been deleted, don't throw an error.
if (err.context?.response?.statusCode !== 404) {
throw err;
if (err.status !== 404) {
params.errorHandler.record("error", fnName, "delete schedule", err.message || "");
}
logger.debug(`Scheduler job ${jobName} not found, continuing deployment.`);
logger.debug(`Scheduler job ${jobName} not found.`);
}
try {
return deleteTopic(topicName);
await deleteTopic(topicName);
helper.printSuccess(fnName, "delete schedule");
} catch (err) {
// If the topic has already been deleted, don't throw an error.
if (err.context?.response?.statusCode !== 404) {
throw err;
if (err.status !== 404) {
params.errorHandler.record("error", fnName, "delete schedule", err.message || "");
}
logger.debug(`Scheduler topic ${topicName} not found, continuing deployment.`);
logger.debug(`Scheduler topic ${topicName} not found.`);
}
};
}
Expand Down Expand Up @@ -233,25 +258,11 @@ export function runRegionalFunctionDeployment(
if (regionalDeployment.functionsToCreate.length) {
const firstFn = regionalDeployment.functionsToCreate.shift()!;
const task = createFunctionTask(params, firstFn!, onPollFn);
return queue
.run(task)
.then(() => {
helper.printSuccess(firstFn.name, "create");
})
.catch((err) => {
params.errorHandler.record("error", firstFn.name, "create", err.message || "");
});
return queue.run(task);
} else if (regionalDeployment.functionsToUpdate.length) {
const firstFn = regionalDeployment.functionsToUpdate.shift()!;
const task = updateFunctionTask(params, firstFn!, onPollFn);
return queue
.run(task)
.then(() => {
helper.printSuccess(firstFn.name, "update");
})
.catch((err) => {
params.errorHandler.record("error", firstFn.name, "update", err.message || "");
});
return queue.run(task);
}
// If there are no functions to create or update in this region, no need to do anything.
return Promise.resolve();
Expand All @@ -263,23 +274,9 @@ function finishRegionalFunctionDeployment(
queue: Queue<() => Promise<any>, void>
): void {
for (const fn of regionalDeployment.functionsToCreate) {
queue
.run(createFunctionTask(params, fn))
.then(() => {
helper.printSuccess(fn.name, "create");
})
.catch((err) => {
params.errorHandler.record("error", fn.name, "create", err.message || "");
});
queue.run(createFunctionTask(params, fn));
}
for (const fn of regionalDeployment.functionsToUpdate) {
queue
.run(updateFunctionTask(params, fn))
.then(() => {
helper.printSuccess(fn.name, "update");
})
.catch((err) => {
params.errorHandler.record("error", fn.name, "update", err.message || "");
});
queue.run(updateFunctionTask(params, fn));
}
}