Skip to content

fix: agentcontainers: fix flake when ctx cancelled while running docker inspect #18526

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 24, 2025
Merged

fix: agentcontainers: fix flake when ctx cancelled while running docker inspect #18526

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c9d3b2c
fix: agentcontainers: fix flake when ctx cancelled while running dock…
johnstcn Jun 24, 2025
6f6a29e
address comments
johnstcn Jun 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions agent/agentcontainers/api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,11 @@ func (api *API) updaterLoop() {
// and anyone looking to interact with the API.
api.logger.Debug(api.ctx, "performing initial containers update")
if err := api.updateContainers(api.ctx); err != nil {
api.logger.Error(api.ctx, "initial containers update failed", slog.Error(err))
if errors.Is(err, context.Canceled) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note that context.Canceled doesn't, by default, trigger test failure. Still, nice to not pollute the error stream.

api.logger.Warn(api.ctx, "initial containers update canceled", slog.Error(err))
} else {
api.logger.Error(api.ctx, "initial containers update failed", slog.Error(err))
}
} else {
api.logger.Debug(api.ctx, "initial containers update complete")
}
Expand All @@ -399,7 +403,11 @@ func (api *API) updaterLoop() {
case api.updateTrigger <- done:
err := <-done
if err != nil {
api.logger.Error(api.ctx, "updater loop ticker failed", slog.Error(err))
if errors.Is(err, context.Canceled) {
api.logger.Warn(api.ctx, "updater loop ticker canceled", slog.Error(err))
} else {
api.logger.Error(api.ctx, "updater loop ticker failed", slog.Error(err))
}
}
default:
api.logger.Debug(api.ctx, "updater loop ticker skipped, update in progress")
Expand Down
10 changes: 10 additions & 0 deletions agent/agentcontainers/containers_dockercli.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,10 @@ func (dcli *dockerCLI) List(ctx context.Context) (codersdk.WorkspaceAgentListCon
// container IDs and returns the parsed output.
// The stderr output is also returned for logging purposes.
func runDockerInspect(ctx context.Context, execer agentexec.Execer, ids ...string) (stdout, stderr []byte, err error) {
if ctx.Err() != nil {
// If the context is done, we don't want to run the command.
return []byte{}, []byte{}, ctx.Err()
}
var stdoutBuf, stderrBuf bytes.Buffer
cmd := execer.CommandContext(ctx, "docker", append([]string{"inspect"}, ids...)...)
cmd.Stdout = &stdoutBuf
Expand All @@ -319,6 +323,12 @@ func runDockerInspect(ctx context.Context, execer agentexec.Execer, ids ...strin
stdout = bytes.TrimSpace(stdoutBuf.Bytes())
stderr = bytes.TrimSpace(stderrBuf.Bytes())
if err != nil {
if ctx.Err() != nil {
// If the context was canceled while running the command,
// return the context error instead of the command error,
// which is likely to be "signal: killed".
return stdout, stderr, ctx.Err()
}
if bytes.Contains(stderr, []byte("No such object:")) {
// This can happen if a container is deleted between the time we check for its existence and the time we inspect it.
return stdout, stderr, nil
Expand Down
Loading