Skip to content

DAOS-18618 pool: fix pool destroy hang during stop #18284

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wangshilong wants to merge 1 commit into
base: master
Choose a base branch
from
+42 −7
Open

DAOS-18618 pool: fix pool destroy hang during stop #18284

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion src/pool/srv_pool_scrub_ult.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*
* (C) Copyright 2021-2022 Intel Corporation.
* (C) Copyright 2025 Hewlett Packard Enterprise Development LP
* (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/
Expand Down Expand Up @@ -344,6 +344,12 @@ scrubbing_ult(void *arg)
while (!dss_ult_exiting(child->spc_scrubbing_req)) {
uint32_t sleep_time = 1000;

if (child->spc_pool->sp_stopping) {
D_INFO(DF_UUID ": pool is stopping, scrubber exiting\n",
DP_UUID(pool_uuid));
break;
}

rc = vos_scrub_pool(&ctx);
if (rc == -DER_SHUTDOWN) {
D_ERROR("tgt_id %d shutting down. Scrubber should quit\n",
Expand Down
41 changes: 35 additions & 6 deletions src/vos/vos_pool_scrub.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*
* (C) Copyright 2020-2024 Intel Corporation.
* (C) Copyright 2025 Hewlett Packard Enterprise Development LP
* (C) Copyright 2025-2026 Hewlett Packard Enterprise Development LP
*
* SPDX-License-Identifier: BSD-2-Clause-Patent
*/
Expand Down Expand Up @@ -161,6 +161,12 @@ sc_cont_is_stopping(struct scrub_ctx *ctx)
return ctx->sc_cont_is_stopping_fn(ctx->sc_cont.scs_cont_src);
}

static inline bool
sc_pool_is_stopping(const struct scrub_ctx *ctx)
{
return ctx->sc_pool != NULL && ctx->sc_pool->sp_stopping;
}

static inline bool
sc_scrub_enabled(struct scrub_ctx *ctx)
{
Expand Down Expand Up @@ -232,6 +238,9 @@ sc_should_start(struct scrub_ctx *ctx)
if (!sc_scrub_enabled(ctx))
return false;

if (sc_pool_is_stopping(ctx))
return false;

if (ctx->sc_pool_scrub_count == 0 || sc_frequency_time_over(ctx)) {
if (ctx->sc_pool->sp_scrub_mode == DAOS_SCRUB_MODE_LAZY) { /* only run if idle */
bool is_idle = sc_is_idle(ctx);
Expand Down Expand Up @@ -803,8 +812,17 @@ cont_iter_scrub_cb(daos_handle_t ih, vos_iter_entry_t *entry,
} else {
rc = sc_cont_setup(ctx, entry);
if (rc != 0) {
/* log error for container, but then keep going */
D_ERROR("Unable to setup the container. "DF_RC"\n", DP_RC(rc));
/* DER_CONT_NONEXIST is expected when a container is deleted;
* log at debug level and skip it. Use error level for any
* other unexpected failure.
*/
if (rc == -DER_CONT_NONEXIST)
D_DEBUG(DB_CSUM,
"Container " DF_UUIDF " removed, skipping: " DF_RC "\n",
DP_UUID(entry->ie_couuid), DP_RC(rc));
else
D_ERROR("Unable to setup the container " DF_UUIDF ": " DF_RC "\n",
DP_UUID(entry->ie_couuid), DP_RC(rc));
return 0;
}

Expand Down Expand Up @@ -868,10 +886,19 @@ cont_iter_is_loaded_cb(daos_handle_t ih, vos_iter_entry_t *entry,
D_ASSERT(type == VOS_ITER_COUUID);

rc = sc_cont_setup(ctx, entry);
if (rc != 0)
return rc;
if (sc_cont_is_stopping(ctx))
if (rc != 0) {
if (rc == -DER_CONT_NONEXIST)
D_DEBUG(DB_CSUM, "Container " DF_UUIDF " removed, skipping: " DF_RC "\n",
DP_UUID(entry->ie_couuid), DP_RC(rc));
else
D_ERROR("Unable to setup container " DF_UUIDF ": " DF_RC "\n",
DP_UUID(entry->ie_couuid), DP_RC(rc));
return 0;
}
if (sc_cont_is_stopping(ctx)) {
sc_cont_teardown(ctx);
return 0;
}

/*
* Is loaded when the properties have been fetched. That way the csummer has been
Expand Down Expand Up @@ -908,6 +935,8 @@ sc_ensure_containers_are_loaded(struct scrub_ctx *ctx)
args.args_found_unloaded_container = false;
rc = vos_iterate(&param, VOS_ITER_COUUID, false, &anchors, NULL,
cont_iter_is_loaded_cb, &args, NULL);
if (sc_pool_is_stopping(ctx))
return 0;
sc_sleep(ctx, 500);
} while (args.args_found_unloaded_container || rc != 0);
ctx->sc_cont_loaded = true;
Expand Down
Loading